diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 83b6cb38520..f3f866d46a1 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-android/pom.xml b/hapi-fhir-android/pom.xml index 87706cf277d..2dd9942ba2a 100644 --- a/hapi-fhir-android/pom.xml +++ b/hapi-fhir-android/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index ed0a9fc6da4..188e9bcbe27 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/PagingIterator.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/PagingIterator.java index eeb9dd233da..4aeb07531f4 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/PagingIterator.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/model/api/PagingIterator.java @@ -26,13 +26,16 @@ import java.util.LinkedList; import java.util.NoSuchElementException; import java.util.function.Consumer; +/** + * This paging iterator only works with already ordered queries + */ public class PagingIterator implements Iterator { public interface PageFetcher { void fetchNextPage(int thePageIndex, int theBatchSize, Consumer theConsumer); } - static final int PAGE_SIZE = 100; + static final int DEFAULT_PAGE_SIZE = 100; private int myPage; @@ -42,8 +45,16 @@ public class PagingIterator implements Iterator { private final PageFetcher myFetcher; + private final int myPageSize; + public PagingIterator(PageFetcher theFetcher) { + this(DEFAULT_PAGE_SIZE, theFetcher); + } + + public PagingIterator(int thePageSize, PageFetcher theFetcher) { + assert thePageSize > 0 : "Page size must be a positive value"; myFetcher = theFetcher; + myPageSize = thePageSize; } @Override @@ -66,9 +77,9 @@ public class PagingIterator implements Iterator { private void fetchNextBatch() { if (!myIsFinished && myCurrentBatch.isEmpty()) { - myFetcher.fetchNextPage(myPage, PAGE_SIZE, myCurrentBatch::add); + myFetcher.fetchNextPage(myPage, myPageSize, myCurrentBatch::add); myPage++; - myIsFinished = myCurrentBatch.size() < PAGE_SIZE; + myIsFinished = myCurrentBatch.size() < myPageSize; } } } diff --git a/hapi-fhir-base/src/test/java/ca/uhn/fhir/model/api/PagingIteratorTest.java b/hapi-fhir-base/src/test/java/ca/uhn/fhir/model/api/PagingIteratorTest.java index 340d7464684..6cd51910a1f 100644 --- a/hapi-fhir-base/src/test/java/ca/uhn/fhir/model/api/PagingIteratorTest.java +++ b/hapi-fhir-base/src/test/java/ca/uhn/fhir/model/api/PagingIteratorTest.java @@ -62,7 +62,7 @@ public class PagingIteratorTest { public void next_fetchTest_fetchesAndReturns() { // 3 cases to make sure we get the edge cases for (int adj : new int[] { -1, 0, 1 }) { - int size = PagingIterator.PAGE_SIZE + adj; + int size = PagingIterator.DEFAULT_PAGE_SIZE + adj; myPagingIterator = createPagingIterator(size); diff --git a/hapi-fhir-bom/pom.xml b/hapi-fhir-bom/pom.xml index f05fea85c67..11be6c4e547 100644 --- a/hapi-fhir-bom/pom.xml +++ b/hapi-fhir-bom/pom.xml @@ -4,7 +4,7 @@ 4.0.0 ca.uhn.hapi.fhir hapi-fhir-bom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT pom HAPI FHIR BOM @@ -12,7 +12,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-checkstyle/pom.xml b/hapi-fhir-checkstyle/pom.xml index 69c0460cccf..edeebb36033 100644 --- a/hapi-fhir-checkstyle/pom.xml +++ b/hapi-fhir-checkstyle/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index ee43373d473..a6301421773 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml index ec3ae13b3f0..5a80f6e818b 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir-cli - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/pom.xml b/hapi-fhir-cli/pom.xml index f7053e7e873..f3ad5743264 100644 --- a/hapi-fhir-cli/pom.xml +++ b/hapi-fhir-cli/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-client-okhttp/pom.xml b/hapi-fhir-client-okhttp/pom.xml index 35617f0402f..4038b5c527e 100644 --- a/hapi-fhir-client-okhttp/pom.xml +++ b/hapi-fhir-client-okhttp/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-client/pom.xml b/hapi-fhir-client/pom.xml index e974a566096..a13e2b42330 100644 --- a/hapi-fhir-client/pom.xml +++ b/hapi-fhir-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-converter/pom.xml b/hapi-fhir-converter/pom.xml index a14196db410..62951f59897 100644 --- a/hapi-fhir-converter/pom.xml +++ b/hapi-fhir-converter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-dist/pom.xml b/hapi-fhir-dist/pom.xml index 3dda194c646..2e78389b1ec 100644 --- a/hapi-fhir-dist/pom.xml +++ b/hapi-fhir-dist/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-docs/pom.xml b/hapi-fhir-docs/pom.xml index 41acfef29b0..b60a2d29aa1 100644 --- a/hapi-fhir-docs/pom.xml +++ b/hapi-fhir-docs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5745-added-ready-state-to-batch2-work-chunks.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5745-added-ready-state-to-batch2-work-chunks.yaml new file mode 100644 index 00000000000..e293b0a5f43 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5745-added-ready-state-to-batch2-work-chunks.yaml @@ -0,0 +1,10 @@ +--- +type: add +issue: 5745 +title: "Added another state to the Batch2 work chunk state machine: `READY`. + This work chunk state will be the initial state on creation. + Once queued for delivery, they will transition to `QUEUED`. + The exception is for ReductionStep chunks (because reduction steps + are not read off of the queue, but executed by the maintenance job + inline. +" diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5767-add-poll-waiting-step-to-batch-jobs.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5767-add-poll-waiting-step-to-batch-jobs.yaml new file mode 100644 index 00000000000..1d14dee8c60 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5767-add-poll-waiting-step-to-batch-jobs.yaml @@ -0,0 +1,9 @@ +--- +type: add +issue: 5767 +title: "Added new `POLL_WAITING` state for WorkChunks in batch jobs. + Also added RetryChunkLaterException for jobs that have steps that + need to be retried at a later time (can be provided optionally to exception). + If a step throws this new exception, it will be set with the new + `POLL_WAITING` status and retried at a later time. +" diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5818-update-batch2-framework-with-gate_waiting-state.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5818-update-batch2-framework-with-gate_waiting-state.yaml new file mode 100644 index 00000000000..197f01bf9f5 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5818-update-batch2-framework-with-gate_waiting-state.yaml @@ -0,0 +1,7 @@ +--- +type: add +issue: 5818 +title: "Added another state to the Batch2 work chunk state machine: `GATE_WAITING`. + This work chunk state will be the initial state on creation for gated jobs. + Once all chunks are completed for the previous step, they will transition to `READY`. +" diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md index b11e7187ce5..e085a33f7e0 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md @@ -47,24 +47,35 @@ stateDiagram-v2 title: Batch2 Job Work Chunk state transitions --- stateDiagram-v2 + state GATE_WAITING + state READY + state REDUCTION_READY state QUEUED state on_receive <> state IN_PROGRESS state ERROR + state POLL_WAITING state execute <> state FAILED state COMPLETED direction LR - [*] --> QUEUED : on create + [*] --> READY : on create - normal or gated jobs first chunks + [*] --> GATE_WAITING : on create - gated jobs for all but the first chunks of the first step + GATE_WAITING --> READY : on gate release - gated + GATE_WAITING --> REDUCTION_READY : on gate release for the final reduction step (all reduction jobs are gated) + QUEUED --> READY : on gate release - gated (for compatibility with legacy QUEUED state up to Hapi-fhir version 7.1) + READY --> QUEUED : placed on kafka (maint.) + POLL_WAITING --> READY : after a poll delay on a POLL_WAITING work chunk has elapsed %% worker processing states - QUEUED --> on_receive : on deque by worker + QUEUED --> on_receive : on deque by worker on_receive --> IN_PROGRESS : start execution IN_PROGRESS --> execute: execute execute --> ERROR : on re-triable error execute --> COMPLETED : success\n maybe trigger instance first_step_finished execute --> FAILED : on unrecoverable \n or too many errors + execute --> POLL_WAITING : job step has throw a RetryChunkLaterException and must be tried again after the provided poll delay %% temporary error state until retry ERROR --> on_receive : exception rollback\n triggers redelivery diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/introduction.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/introduction.md index ce8dbc4a1f0..1c3bb485c21 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/introduction.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/introduction.md @@ -19,36 +19,54 @@ A HAPI-FHIR batch job definition consists of a job name, version, parameter json After a job has been defined, *instances* of that job can be submitted for batch processing by populating a `JobInstanceStartRequest` with the job name and job parameters json and then submitting that request to the Batch Job Coordinator. The Batch Job Coordinator will then store two records in the database: -- Job Instance with status QUEUED: that is the parent record for all data concerning this job -- Batch Work Chunk with status QUEUED: this describes the first "chunk" of work required for this job. The first Batch Work Chunk contains no data. +- Job Instance with status `QUEUED`: that is the parent record for all data concerning this job +- Batch Work Chunk with status `READY`: this describes the first "chunk" of work required for this job. The first Batch Work Chunk contains no data. -Lastly the Batch Job Coordinator publishes a message to the Batch Notification Message Channel (named `batch2-work-notification`) to inform worker threads that this first chunk of work is now ready for processing. +### The Maintenance Job -### Job Processing - First Step +A Scheduled Job runs periodically (once a minute). For each Job Instance in the database, it: -HAPI-FHIR Batch Jobs run based on job notification messages. The process is kicked off by the first chunk of work. When this notification message arrives, the message handler makes a single call to the first step defined in the job definition, passing in the job parameters as input. +1. Calculates job progress (% of work chunks in `COMPLETE` status). If the job is finished, purges any left over work chunks still in the database. +1. Moves all `POLL_WAITING` work chunks to `READY` if their `nextPollTime` has expired. +1. Calculates job progress (% of work chunks in `COMPLETE` status). If the job is finished, purges any leftover work chunks still in the database. +1. Cleans up any complete, failed, or cancelled jobs that need to be removed. +1. When the current step is complete, moves any gated jobs onto their next step and updates all chunks in `GATE_WAITING` to `READY`. If the the job is being moved to its final reduction step, chunks are moved from `GATE_WAITING` to `REDUCTION_READY`. +1. If the final step of a gated job is a reduction step, a reduction step execution will be triggered. All workchunks for the job in `REDUCTION_READY` will be consumed at this point. +1. Moves all `READY` work chunks into the `QUEUED` state and publishes a message to the Batch Notification Message Channel to inform worker threads that a work chunk is now ready for processing. \* -The handler then does the following: -1. Change the work chunk status from QUEUED to IN_PROGRESS -2. Change the Job Instance status from QUEUED to IN_PROGRESS -3. If the Job Instance is cancelled, change the status to CANCELLED and abort processing. -4. The first step of the job definition is executed with the job parameters -5. This step creates new work chunks. For each work chunk it creates, it json serializes the work chunk data, stores it in the database, and publishes a new message to the Batch Notification Message Channel to notify worker threads that there are new work chunks waiting to be processed. -6. If the step succeeded, the work chunk status is changed from IN_PROGRESS to COMPLETED, and the data it contained is deleted. -7. If the step failed, the work chunk status is changed from IN_PROGRESS to either ERRORED or FAILED depending on the severity of the error. +\* An exception is for the final reduction step, where work chunks are not published to the Batch Notification Message Channel, +but instead processed inline. -### Job Processing - Middle steps +### Batch Notification Message Handler -Middle Steps in the job definition are executed in the same way, except instead of only using the Job Parameters as input, they use both the Job Parameters and the Work Chunk data produced from the previous step. +HAPI-FHIR Batch Jobs run based on job notification messages of the Batch Notification Message Channel (named `batch2-work-notification`). -### Job Processing - Final Step +When a notification message arrives, the handler does the following: + +1. Change the work chunk status from `QUEUED` to `IN_PROGRESS` +1. Change the Job Instance status from `QUEUED` to `IN_PROGRESS` +1. If the Job Instance is cancelled, change the status to `CANCELLED` and abort processing +1. If the step creates new work chunks, each work chunk will be created in either the `GATE_WAITING` state (for gated jobs) or `READY` state (for non-gated jobs) and will be handled in the next maintenance job pass. +1. If the step succeeds, the work chunk status is changed from `IN_PROGRESS` to `COMPLETED`, and the data it contained is deleted. +1. If the step throws a `RetryChunkLaterException`, the work chunk status is changed from `IN_PROGRESS` to `POLL_WAITING`, and a `nextPollTime` value will be set. +1. If the step fails, the work chunk status is changed from `IN_PROGRESS` to either `ERRORED` or `FAILED`, depending on the severity of the error. + +### First Step + +The first step in a job definition is executed with just the job parameters. + +### Middle steps + +Middle Steps in the job definition are executed using the initial Job Parameters and the Work Chunk data produced from the previous step. + +### Final Step The final step operates the same way as the middle steps, except it does not produce any new work chunks. ### Gated Execution -If a Job Definition is set to having Gated Execution, then all work chunks for one step must be COMPLETED before any work chunks for the next step may begin. +If a Job Definition is set to having Gated Execution, then all work chunks for a step must be `COMPLETED` before any work chunks for the next step may begin. ### Job Instance Completion -A Batch Job Maintenance Service runs every minute to monitor the status of all Job Instances and the Job Instance is transitioned to either COMPLETED, ERRORED or FAILED according to the status of all outstanding work chunks for that job instance. If the job instance is still IN_PROGRESS this maintenance service also estimates the time remaining to complete the job. +A Batch Job Maintenance Service runs every minute to monitor the status of all Job Instances and the Job Instance is transitioned to either `COMPLETED`, `ERRORED` or `FAILED` according to the status of all outstanding work chunks for that job instance. If the job instance is still `IN_PROGRESS` this maintenance service also estimates the time remaining to complete the job. diff --git a/hapi-fhir-jacoco/pom.xml b/hapi-fhir-jacoco/pom.xml index 2079a6bb970..3ba709907e0 100644 --- a/hapi-fhir-jacoco/pom.xml +++ b/hapi-fhir-jacoco/pom.xml @@ -11,7 +11,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jaxrsserver-base/pom.xml b/hapi-fhir-jaxrsserver-base/pom.xml index 7077325c2c6..a36f270f1ca 100644 --- a/hapi-fhir-jaxrsserver-base/pom.xml +++ b/hapi-fhir-jaxrsserver-base/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpa/pom.xml b/hapi-fhir-jpa/pom.xml index b7dd29597aa..fd28ff49ed0 100644 --- a/hapi-fhir-jpa/pom.xml +++ b/hapi-fhir-jpa/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/IHapiScheduler.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/IHapiScheduler.java index f2084bfa7c8..f9cb5e6c020 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/IHapiScheduler.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/IHapiScheduler.java @@ -37,6 +37,17 @@ public interface IHapiScheduler { void logStatusForUnitTest(); + /** + * Pauses this scheduler (and thus all scheduled jobs). + * To restart call {@link #unpause()} + */ + void pause(); + + /** + * Restarts this scheduler after {@link #pause()} + */ + void unpause(); + void scheduleJob(long theIntervalMillis, ScheduledJobDefinition theJobDefinition); Set getJobKeysForUnitTest() throws SchedulerException; diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/ISchedulerService.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/ISchedulerService.java index c058198e03c..5ff1057937c 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/ISchedulerService.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/model/sched/ISchedulerService.java @@ -32,6 +32,20 @@ public interface ISchedulerService { void logStatusForUnitTest(); + /** + * Pauses the scheduler so no new jobs will run. + * Useful in tests when cleanup needs to happen but scheduled jobs may + * be running + */ + @VisibleForTesting + void pause(); + + /** + * Restarts the scheduler after a previous call to {@link #pause()}. + */ + @VisibleForTesting + void unpause(); + /** * This task will execute locally (and should execute on all nodes of the cluster if there is a cluster) * @param theIntervalMillis How many milliseconds between passes should this job run @@ -52,6 +66,9 @@ public interface ISchedulerService { @VisibleForTesting Set getClusteredJobKeysForUnitTest() throws SchedulerException; + @VisibleForTesting + boolean isSchedulingDisabled(); + boolean isStopping(); /** diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseHapiScheduler.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseHapiScheduler.java index 916bebe93fa..f8def318609 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseHapiScheduler.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseHapiScheduler.java @@ -29,6 +29,7 @@ import com.google.common.collect.Sets; import jakarta.annotation.Nonnull; import org.apache.commons.lang3.Validate; import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; import org.quartz.JobKey; import org.quartz.ScheduleBuilder; import org.quartz.Scheduler; @@ -44,11 +45,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.quartz.SchedulerFactoryBean; +import java.util.List; import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + public abstract class BaseHapiScheduler implements IHapiScheduler { private static final Logger ourLog = LoggerFactory.getLogger(BaseHapiScheduler.class); @@ -151,6 +155,42 @@ public abstract class BaseHapiScheduler implements IHapiScheduler { } } + public void pause() { + int delay = 100; + String errorMsg = null; + Throwable ex = null; + try { + int count = 0; + myScheduler.standby(); + while (count < 3) { + if (!hasRunningJobs()) { + break; + } + Thread.sleep(delay); + count++; + } + if (count >= 3) { + errorMsg = "Scheduler on standby. But after " + (count + 1) * delay + + " ms there are still jobs running. Execution will continue, but may cause bugs."; + } + } catch (Exception x) { + ex = x; + errorMsg = "Failed to set to standby. Execution will continue, but may cause bugs."; + } + + if (isNotBlank(errorMsg)) { + if (ex != null) { + ourLog.warn(errorMsg, ex); + } else { + ourLog.warn(errorMsg); + } + } + } + + public void unpause() { + start(); + } + @Override public void clear() throws SchedulerException { myScheduler.clear(); @@ -168,6 +208,16 @@ public abstract class BaseHapiScheduler implements IHapiScheduler { } } + private boolean hasRunningJobs() { + try { + List currentlyExecutingJobs = myScheduler.getCurrentlyExecutingJobs(); + ourLog.info("Checking for running jobs. Found {} running.", currentlyExecutingJobs); + return !currentlyExecutingJobs.isEmpty(); + } catch (SchedulerException ex) { + throw new RuntimeException(Msg.code(2521) + " Failed during check for scheduled jobs", ex); + } + } + @Override public void scheduleJob(long theIntervalMillis, ScheduledJobDefinition theJobDefinition) { Validate.isTrue(theIntervalMillis >= 100); diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseSchedulerServiceImpl.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseSchedulerServiceImpl.java index 358aa408176..89036097ecf 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseSchedulerServiceImpl.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/BaseSchedulerServiceImpl.java @@ -136,7 +136,7 @@ public abstract class BaseSchedulerServiceImpl implements ISchedulerService { return retval; } - private boolean isSchedulingDisabled() { + public boolean isSchedulingDisabled() { return !isLocalSchedulingEnabled() || isSchedulingDisabledForUnitTests(); } @@ -198,6 +198,18 @@ public abstract class BaseSchedulerServiceImpl implements ISchedulerService { myClusteredScheduler.logStatusForUnitTest(); } + @Override + public void pause() { + myLocalScheduler.pause(); + myClusteredScheduler.pause(); + } + + @Override + public void unpause() { + myLocalScheduler.unpause(); + myClusteredScheduler.unpause(); + } + @Override public void scheduleLocalJob(long theIntervalMillis, ScheduledJobDefinition theJobDefinition) { scheduleJob("local", myLocalScheduler, theIntervalMillis, theJobDefinition); diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/HapiNullScheduler.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/HapiNullScheduler.java index 349174eacfd..77c7217e850 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/HapiNullScheduler.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/sched/HapiNullScheduler.java @@ -53,6 +53,16 @@ public class HapiNullScheduler implements IHapiScheduler { @Override public void logStatusForUnitTest() {} + @Override + public void pause() { + // nothing to do + } + + @Override + public void unpause() { + // nothing to do + } + @Override public void scheduleJob(long theIntervalMillis, ScheduledJobDefinition theJobDefinition) { ourLog.debug("Skipping scheduling job {} since scheduling is disabled", theJobDefinition.getId()); diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 6cf0698fa2e..d4995828336 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java index c2db708638a..51698299f79 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java @@ -123,6 +123,8 @@ class JobInstanceUtil { retVal.setErrorMessage(theEntity.getErrorMessage()); retVal.setErrorCount(theEntity.getErrorCount()); retVal.setRecordsProcessed(theEntity.getRecordsProcessed()); + retVal.setNextPollTime(theEntity.getNextPollTime()); + retVal.setPollAttempts(theEntity.getPollAttempts()); // note: may be null out if queried NoData retVal.setData(theEntity.getSerializedData()); retVal.setWarningMessage(theEntity.getWarningMessage()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java index 056a546b5c9..f73a88570f3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java @@ -24,6 +24,7 @@ import ca.uhn.fhir.batch2.config.BaseBatch2Config; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; +import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkMetadataViewRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import jakarta.persistence.EntityManager; @@ -39,12 +40,14 @@ public class JpaBatch2Config extends BaseBatch2Config { public IJobPersistence batch2JobInstancePersister( IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, + IBatch2WorkChunkMetadataViewRepository theWorkChunkMetadataViewRepo, IHapiTransactionService theTransactionService, EntityManager theEntityManager, IInterceptorBroadcaster theInterceptorBroadcaster) { return new JpaJobPersistenceImpl( theJobInstanceRepository, theWorkChunkRepository, + theWorkChunkMetadataViewRepo, theTransactionService, theEntityManager, theInterceptorBroadcaster); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java index 48140f8477c..5ea89d2adb4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java @@ -28,16 +28,19 @@ import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkMetadata; import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.interceptor.api.HookParams; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; +import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkMetadataViewRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; +import ca.uhn.fhir.jpa.entity.Batch2WorkChunkMetadataView; import ca.uhn.fhir.model.api.PagingIterator; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; @@ -64,7 +67,10 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronizationManager; +import java.time.Instant; +import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -85,6 +91,7 @@ public class JpaJobPersistenceImpl implements IJobPersistence { private final IBatch2JobInstanceRepository myJobInstanceRepository; private final IBatch2WorkChunkRepository myWorkChunkRepository; + private final IBatch2WorkChunkMetadataViewRepository myWorkChunkMetadataViewRepo; private final EntityManager myEntityManager; private final IHapiTransactionService myTransactionService; private final IInterceptorBroadcaster myInterceptorBroadcaster; @@ -95,13 +102,15 @@ public class JpaJobPersistenceImpl implements IJobPersistence { public JpaJobPersistenceImpl( IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, + IBatch2WorkChunkMetadataViewRepository theWorkChunkMetadataViewRepo, IHapiTransactionService theTransactionService, EntityManager theEntityManager, IInterceptorBroadcaster theInterceptorBroadcaster) { - Validate.notNull(theJobInstanceRepository); - Validate.notNull(theWorkChunkRepository); + Validate.notNull(theJobInstanceRepository, "theJobInstanceRepository"); + Validate.notNull(theWorkChunkRepository, "theWorkChunkRepository"); myJobInstanceRepository = theJobInstanceRepository; myWorkChunkRepository = theWorkChunkRepository; + myWorkChunkMetadataViewRepo = theWorkChunkMetadataViewRepo; myTransactionService = theTransactionService; myEntityManager = theEntityManager; myInterceptorBroadcaster = theInterceptorBroadcaster; @@ -120,23 +129,46 @@ public class JpaJobPersistenceImpl implements IJobPersistence { entity.setSerializedData(theBatchWorkChunk.serializedData); entity.setCreateTime(new Date()); entity.setStartTime(new Date()); - entity.setStatus(WorkChunkStatusEnum.QUEUED); + entity.setStatus(getOnCreateStatus(theBatchWorkChunk)); + ourLog.debug("Create work chunk {}/{}/{}", entity.getInstanceId(), entity.getId(), entity.getTargetStepId()); ourLog.trace( "Create work chunk data {}/{}: {}", entity.getInstanceId(), entity.getId(), entity.getSerializedData()); myTransactionService.withSystemRequestOnDefaultPartition().execute(() -> myWorkChunkRepository.save(entity)); + return entity.getId(); } + /** + * Gets the initial onCreate state for the given workchunk. + * Gated job chunks start in GATE_WAITING; they will be transitioned to READY during maintenance pass when all + * chunks in the previous step are COMPLETED. + * Non gated job chunks start in READY + */ + private static WorkChunkStatusEnum getOnCreateStatus(WorkChunkCreateEvent theBatchWorkChunk) { + if (theBatchWorkChunk.isGatedExecution) { + return WorkChunkStatusEnum.GATE_WAITING; + } else { + return WorkChunkStatusEnum.READY; + } + } + @Override @Transactional(propagation = Propagation.REQUIRED) public Optional onWorkChunkDequeue(String theChunkId) { + // take a lock on the chunk id to ensure that the maintenance run isn't doing anything. + Batch2WorkChunkEntity chunkLock = + myEntityManager.find(Batch2WorkChunkEntity.class, theChunkId, LockModeType.PESSIMISTIC_WRITE); + // remove from the current state to avoid stale data. + myEntityManager.detach(chunkLock); + // NOTE: Ideally, IN_PROGRESS wouldn't be allowed here. On chunk failure, we probably shouldn't be allowed. // But how does re-run happen if k8s kills a processor mid run? List priorStates = List.of(WorkChunkStatusEnum.QUEUED, WorkChunkStatusEnum.ERRORED, WorkChunkStatusEnum.IN_PROGRESS); int rowsModified = myWorkChunkRepository.updateChunkStatusForStart( theChunkId, new Date(), WorkChunkStatusEnum.IN_PROGRESS, priorStates); + if (rowsModified == 0) { ourLog.info("Attempting to start chunk {} but it was already started.", theChunkId); return Optional.empty(); @@ -288,6 +320,22 @@ public class JpaJobPersistenceImpl implements IJobPersistence { .collect(Collectors.toList())); } + @Override + public void enqueueWorkChunkForProcessing(String theChunkId, Consumer theCallback) { + int updated = myWorkChunkRepository.updateChunkStatus( + theChunkId, WorkChunkStatusEnum.READY, WorkChunkStatusEnum.QUEUED); + theCallback.accept(updated); + } + + @Override + public int updatePollWaitingChunksForJobIfReady(String theInstanceId) { + return myWorkChunkRepository.updateWorkChunksForPollWaiting( + theInstanceId, + Date.from(Instant.now()), + Set.of(WorkChunkStatusEnum.POLL_WAITING), + WorkChunkStatusEnum.READY); + } + @Override @Transactional(propagation = Propagation.REQUIRES_NEW) public List fetchRecentInstances(int thePageSize, int thePageIndex) { @@ -333,6 +381,16 @@ public class JpaJobPersistenceImpl implements IJobPersistence { }); } + @Override + public void onWorkChunkPollDelay(String theChunkId, Date theDeadline) { + int updated = myWorkChunkRepository.updateWorkChunkNextPollTime( + theChunkId, WorkChunkStatusEnum.POLL_WAITING, Set.of(WorkChunkStatusEnum.IN_PROGRESS), theDeadline); + + if (updated != 1) { + ourLog.warn("Expected to update 1 work chunk's poll delay; but found {}", updated); + } + } + @Override public void onWorkChunkFailed(String theChunkId, String theErrorMessage) { ourLog.info("Marking chunk {} as failed with message: {}", theChunkId, theErrorMessage); @@ -383,24 +441,23 @@ public class JpaJobPersistenceImpl implements IJobPersistence { } @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public boolean canAdvanceInstanceToNextStep(String theInstanceId, String theCurrentStepId) { + public Set getDistinctWorkChunkStatesForJobAndStep( + String theInstanceId, String theCurrentStepId) { + if (getRunningJob(theInstanceId) == null) { + return Collections.unmodifiableSet(new HashSet<>()); + } + return myWorkChunkRepository.getDistinctStatusesForStep(theInstanceId, theCurrentStepId); + } + + private Batch2JobInstanceEntity getRunningJob(String theInstanceId) { Optional instance = myJobInstanceRepository.findById(theInstanceId); if (instance.isEmpty()) { - return false; + return null; } if (instance.get().getStatus().isEnded()) { - return false; + return null; } - Set statusesForStep = - myWorkChunkRepository.getDistinctStatusesForStep(theInstanceId, theCurrentStepId); - - ourLog.debug( - "Checking whether gated job can advanced to next step. [instanceId={}, currentStepId={}, statusesForStep={}]", - theInstanceId, - theCurrentStepId, - statusesForStep); - return statusesForStep.isEmpty() || statusesForStep.equals(Set.of(WorkChunkStatusEnum.COMPLETED)); + return instance.get(); } private void fetchChunks( @@ -428,18 +485,16 @@ public class JpaJobPersistenceImpl implements IJobPersistence { } @Override - public List fetchAllChunkIdsForStepWithStatus( - String theInstanceId, String theStepId, WorkChunkStatusEnum theStatusEnum) { - return myTransactionService - .withSystemRequest() - .withPropagation(Propagation.REQUIRES_NEW) - .execute(() -> myWorkChunkRepository.fetchAllChunkIdsForStepWithStatus( - theInstanceId, theStepId, theStatusEnum)); + public void updateInstanceUpdateTime(String theInstanceId) { + myJobInstanceRepository.updateInstanceUpdateTime(theInstanceId, new Date()); } @Override - public void updateInstanceUpdateTime(String theInstanceId) { - myJobInstanceRepository.updateInstanceUpdateTime(theInstanceId, new Date()); + public WorkChunk createWorkChunk(WorkChunk theWorkChunk) { + if (theWorkChunk.getId() == null) { + theWorkChunk.setId(UUID.randomUUID().toString()); + } + return toChunk(myWorkChunkRepository.save(Batch2WorkChunkEntity.fromWorkChunk(theWorkChunk))); } /** @@ -458,6 +513,15 @@ public class JpaJobPersistenceImpl implements IJobPersistence { .map(this::toChunk); } + @Override + public Page fetchAllWorkChunkMetadataForJobInStates( + Pageable thePageable, String theInstanceId, Set theStates) { + Page page = + myWorkChunkMetadataViewRepo.fetchWorkChunkMetadataForJobInStates(thePageable, theInstanceId, theStates); + + return page.map(Batch2WorkChunkMetadataView::toChunkMetadata); + } + @Override public boolean updateInstance(String theInstanceId, JobInstanceUpdateCallback theModifier) { Batch2JobInstanceEntity instanceEntity = @@ -542,4 +606,45 @@ public class JpaJobPersistenceImpl implements IJobPersistence { myInterceptorBroadcaster.callHooks(Pointcut.STORAGE_PRESTORAGE_BATCH_JOB_CREATE, params); } } + + @Override + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean advanceJobStepAndUpdateChunkStatus( + String theJobInstanceId, String theNextStepId, boolean theIsReductionStep) { + boolean changed = updateInstance(theJobInstanceId, instance -> { + if (instance.getCurrentGatedStepId().equals(theNextStepId)) { + // someone else beat us here. No changes + return false; + } + ourLog.debug("Moving gated instance {} to the next step {}.", theJobInstanceId, theNextStepId); + instance.setCurrentGatedStepId(theNextStepId); + return true; + }); + + if (changed) { + ourLog.debug( + "Updating chunk status from GATE_WAITING to READY for gated instance {} in step {}.", + theJobInstanceId, + theNextStepId); + WorkChunkStatusEnum nextStep = + theIsReductionStep ? WorkChunkStatusEnum.REDUCTION_READY : WorkChunkStatusEnum.READY; + // when we reach here, the current step id is equal to theNextStepId + // Up to 7.1, gated jobs' work chunks are created in status QUEUED but not actually queued for the + // workers. + // In order to keep them compatible, turn QUEUED chunks into READY, too. + // TODO: 'QUEUED' from the IN clause will be removed after 7.6.0. + int numChanged = myWorkChunkRepository.updateAllChunksForStepWithStatus( + theJobInstanceId, + theNextStepId, + List.of(WorkChunkStatusEnum.GATE_WAITING, WorkChunkStatusEnum.QUEUED), + nextStep); + ourLog.debug( + "Updated {} chunks of gated instance {} for step {} from fake QUEUED to READY.", + numChanged, + theJobInstanceId, + theNextStepId); + } + + return changed; + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkMetadataViewRepository.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkMetadataViewRepository.java new file mode 100644 index 00000000000..2c759143ef6 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkMetadataViewRepository.java @@ -0,0 +1,21 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.fhir.jpa.entity.Batch2WorkChunkMetadataView; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.Collection; + +public interface IBatch2WorkChunkMetadataViewRepository extends JpaRepository { + + @Query("SELECT v FROM Batch2WorkChunkMetadataView v WHERE v.myInstanceId = :instanceId AND v.myStatus IN :states " + + " ORDER BY v.myInstanceId, v.myTargetStepId, v.myStatus, v.mySequence, v.myId ASC") + Page fetchWorkChunkMetadataForJobInStates( + Pageable thePageRequest, + @Param("instanceId") String theInstanceId, + @Param("states") Collection theStates); +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java index 053fc35f89e..0dc9243290e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java @@ -49,7 +49,8 @@ public interface IBatch2WorkChunkRepository @Query("SELECT new Batch2WorkChunkEntity(" + "e.myId, e.mySequence, e.myJobDefinitionId, e.myJobDefinitionVersion, e.myInstanceId, e.myTargetStepId, e.myStatus," + "e.myCreateTime, e.myStartTime, e.myUpdateTime, e.myEndTime," - + "e.myErrorMessage, e.myErrorCount, e.myRecordsProcessed, e.myWarningMessage" + + "e.myErrorMessage, e.myErrorCount, e.myRecordsProcessed, e.myWarningMessage," + + "e.myNextPollTime, e.myPollAttempts" + ") FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId ORDER BY e.mySequence ASC, e.myId ASC") List fetchChunksNoData(Pageable thePageRequest, @Param("instanceId") String theInstanceId); @@ -75,6 +76,24 @@ public interface IBatch2WorkChunkRepository @Param("status") WorkChunkStatusEnum theInProgress, @Param("warningMessage") String theWarningMessage); + @Modifying + @Query( + "UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myNextPollTime = :nextPollTime, e.myPollAttempts = e.myPollAttempts + 1 WHERE e.myId = :id AND e.myStatus IN(:states)") + int updateWorkChunkNextPollTime( + @Param("id") String theChunkId, + @Param("status") WorkChunkStatusEnum theStatus, + @Param("states") Set theInitialStates, + @Param("nextPollTime") Date theNextPollTime); + + @Modifying + @Query( + "UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myNextPollTime = null WHERE e.myInstanceId = :instanceId AND e.myStatus IN(:states) AND e.myNextPollTime <= :pollTime") + int updateWorkChunksForPollWaiting( + @Param("instanceId") String theInstanceId, + @Param("pollTime") Date theTime, + @Param("states") Set theInitialStates, + @Param("status") WorkChunkStatusEnum theNewStatus); + @Modifying @Query( "UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.mySerializedData = null, e.mySerializedDataVc = null, e.myErrorMessage = :em WHERE e.myId IN(:ids)") @@ -102,6 +121,22 @@ public interface IBatch2WorkChunkRepository @Param("status") WorkChunkStatusEnum theInProgress, @Param("startStatuses") Collection theStartStatuses); + @Modifying + @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :newStatus WHERE e.myId = :id AND e.myStatus = :oldStatus") + int updateChunkStatus( + @Param("id") String theChunkId, + @Param("oldStatus") WorkChunkStatusEnum theOldStatus, + @Param("newStatus") WorkChunkStatusEnum theNewStatus); + + @Modifying + @Query( + "UPDATE Batch2WorkChunkEntity e SET e.myStatus = :newStatus WHERE e.myInstanceId = :instanceId AND e.myTargetStepId = :stepId AND e.myStatus IN ( :oldStatuses )") + int updateAllChunksForStepWithStatus( + @Param("instanceId") String theInstanceId, + @Param("stepId") String theStepId, + @Param("oldStatuses") List theOldStatuses, + @Param("newStatus") WorkChunkStatusEnum theNewStatus); + @Modifying @Query("DELETE FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId") int deleteAllForInstance(@Param("instanceId") String theInstanceId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java index 42281ac475f..ff34c74b6fd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java @@ -19,6 +19,7 @@ */ package ca.uhn.fhir.jpa.entity; +import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import jakarta.persistence.Basic; import jakarta.persistence.Column; @@ -50,7 +51,10 @@ import static org.apache.commons.lang3.StringUtils.left; @Entity @Table( name = "BT2_WORK_CHUNK", - indexes = {@Index(name = "IDX_BT2WC_II_SEQ", columnList = "INSTANCE_ID,SEQ")}) + indexes = { + @Index(name = "IDX_BT2WC_II_SEQ", columnList = "INSTANCE_ID,SEQ"), + @Index(name = "IDX_BT2WC_II_SI_S_SEQ_ID", columnList = "INSTANCE_ID,TGT_STEP_ID,STAT,SEQ,ID") + }) public class Batch2WorkChunkEntity implements Serializable { public static final int ERROR_MSG_MAX_LENGTH = 500; @@ -125,6 +129,19 @@ public class Batch2WorkChunkEntity implements Serializable { @Column(name = "WARNING_MSG", length = WARNING_MSG_MAX_LENGTH, nullable = true) private String myWarningMessage; + /** + * The next time the work chunk can attempt to rerun its work step. + */ + @Column(name = "NEXT_POLL_TIME", nullable = true) + @Temporal(TemporalType.TIMESTAMP) + private Date myNextPollTime; + + /** + * The number of times the work chunk has had its state set back to POLL_WAITING. + */ + @Column(name = "POLL_ATTEMPTS", nullable = true) + private int myPollAttempts; + /** * Default constructor for Hibernate. */ @@ -148,7 +165,9 @@ public class Batch2WorkChunkEntity implements Serializable { String theErrorMessage, int theErrorCount, Integer theRecordsProcessed, - String theWarningMessage) { + String theWarningMessage, + Date theNextPollTime, + Integer thePollAttempts) { myId = theId; mySequence = theSequence; myJobDefinitionId = theJobDefinitionId; @@ -164,6 +183,32 @@ public class Batch2WorkChunkEntity implements Serializable { myErrorCount = theErrorCount; myRecordsProcessed = theRecordsProcessed; myWarningMessage = theWarningMessage; + myNextPollTime = theNextPollTime; + myPollAttempts = thePollAttempts; + } + + public static Batch2WorkChunkEntity fromWorkChunk(WorkChunk theWorkChunk) { + Batch2WorkChunkEntity entity = new Batch2WorkChunkEntity( + theWorkChunk.getId(), + theWorkChunk.getSequence(), + theWorkChunk.getJobDefinitionId(), + theWorkChunk.getJobDefinitionVersion(), + theWorkChunk.getInstanceId(), + theWorkChunk.getTargetStepId(), + theWorkChunk.getStatus(), + theWorkChunk.getCreateTime(), + theWorkChunk.getStartTime(), + theWorkChunk.getUpdateTime(), + theWorkChunk.getEndTime(), + theWorkChunk.getErrorMessage(), + theWorkChunk.getErrorCount(), + theWorkChunk.getRecordsProcessed(), + theWorkChunk.getWarningMessage(), + theWorkChunk.getNextPollTime(), + theWorkChunk.getPollAttempts()); + entity.setSerializedData(theWorkChunk.getData()); + + return entity; } public int getErrorCount() { @@ -299,6 +344,22 @@ public class Batch2WorkChunkEntity implements Serializable { myInstanceId = theInstanceId; } + public Date getNextPollTime() { + return myNextPollTime; + } + + public void setNextPollTime(Date theNextPollTime) { + myNextPollTime = theNextPollTime; + } + + public int getPollAttempts() { + return myPollAttempts; + } + + public void setPollAttempts(int thePollAttempts) { + myPollAttempts = thePollAttempts; + } + @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE) @@ -318,6 +379,8 @@ public class Batch2WorkChunkEntity implements Serializable { .append("status", myStatus) .append("errorMessage", myErrorMessage) .append("warningMessage", myWarningMessage) + .append("nextPollTime", myNextPollTime) + .append("pollAttempts", myPollAttempts) .toString(); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkMetadataView.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkMetadataView.java new file mode 100644 index 00000000000..4034a13f7cd --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkMetadataView.java @@ -0,0 +1,123 @@ +package ca.uhn.fhir.jpa.entity; + +import ca.uhn.fhir.batch2.model.WorkChunkMetadata; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import org.hibernate.annotations.Immutable; +import org.hibernate.annotations.Subselect; + +import java.io.Serializable; + +import static ca.uhn.fhir.batch2.model.JobDefinition.ID_MAX_LENGTH; + +/** + * A view for a Work Chunk that contains only the most necessary information + * to satisfy the no-data path. + */ +@Entity +@Immutable +@Subselect("SELECT e.id as id, " + + " e.seq as seq," + + " e.stat as state, " + + " e.instance_id as instance_id, " + + " e.definition_id as job_definition_id, " + + " e.definition_ver as job_definition_version, " + + " e.tgt_step_id as target_step_id " + + "FROM BT2_WORK_CHUNK e") +public class Batch2WorkChunkMetadataView implements Serializable { + + @Id + @Column(name = "ID", length = ID_MAX_LENGTH) + private String myId; + + @Column(name = "SEQ", nullable = false) + private int mySequence; + + @Column(name = "STATE", length = ID_MAX_LENGTH, nullable = false) + @Enumerated(EnumType.STRING) + private WorkChunkStatusEnum myStatus; + + @Column(name = "INSTANCE_ID", length = ID_MAX_LENGTH, nullable = false) + private String myInstanceId; + + @Column(name = "JOB_DEFINITION_ID", length = ID_MAX_LENGTH, nullable = false) + private String myJobDefinitionId; + + @Column(name = "JOB_DEFINITION_VERSION", nullable = false) + private int myJobDefinitionVersion; + + @Column(name = "TARGET_STEP_ID", length = ID_MAX_LENGTH, nullable = false) + private String myTargetStepId; + + public String getId() { + return myId; + } + + public void setId(String theId) { + myId = theId; + } + + public int getSequence() { + return mySequence; + } + + public void setSequence(int theSequence) { + mySequence = theSequence; + } + + public WorkChunkStatusEnum getStatus() { + return myStatus; + } + + public void setStatus(WorkChunkStatusEnum theStatus) { + myStatus = theStatus; + } + + public String getInstanceId() { + return myInstanceId; + } + + public void setInstanceId(String theInstanceId) { + myInstanceId = theInstanceId; + } + + public String getJobDefinitionId() { + return myJobDefinitionId; + } + + public void setJobDefinitionId(String theJobDefinitionId) { + myJobDefinitionId = theJobDefinitionId; + } + + public int getJobDefinitionVersion() { + return myJobDefinitionVersion; + } + + public void setJobDefinitionVersion(int theJobDefinitionVersion) { + myJobDefinitionVersion = theJobDefinitionVersion; + } + + public String getTargetStepId() { + return myTargetStepId; + } + + public void setTargetStepId(String theTargetStepId) { + myTargetStepId = theTargetStepId; + } + + public WorkChunkMetadata toChunkMetadata() { + WorkChunkMetadata metadata = new WorkChunkMetadata(); + metadata.setId(getId()); + metadata.setInstanceId(getInstanceId()); + metadata.setSequence(getSequence()); + metadata.setStatus(getStatus()); + metadata.setJobDefinitionId(getJobDefinitionId()); + metadata.setJobDefinitionVersion(getJobDefinitionVersion()); + metadata.setTargetStepId(getTargetStepId()); + return metadata; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index 86e27081679..3c421278e10 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -293,6 +293,23 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { // This fix will work for MSSQL or Oracle. version.addTask(new ForceIdMigrationFixTask(version.getRelease(), "20231222.1")); + + // add index to Batch2WorkChunkEntity + Builder.BuilderWithTableName workChunkTable = version.onTable("BT2_WORK_CHUNK"); + + workChunkTable + .addIndex("20240321.1", "IDX_BT2WC_II_SI_S_SEQ_ID") + .unique(false) + .withColumns("INSTANCE_ID", "TGT_STEP_ID", "STAT", "SEQ", "ID"); + + // add columns to Batch2WorkChunkEntity + Builder.BuilderWithTableName batch2WorkChunkTable = version.onTable("BT2_WORK_CHUNK"); + + batch2WorkChunkTable + .addColumn("20240322.1", "NEXT_POLL_TIME") + .nullable() + .type(ColumnTypeEnum.DATE_TIMESTAMP); + batch2WorkChunkTable.addColumn("20240322.2", "POLL_ATTEMPTS").nullable().type(ColumnTypeEnum.INT); } private void init680_Part2() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java index ba013f714bc..14b21559d52 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java @@ -4,6 +4,7 @@ import ca.uhn.fhir.batch2.api.JobOperationResultJson; import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; @@ -31,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkDataExportJobSchedulingHelperImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkDataExportJobSchedulingHelperImplTest.java index f7c90efb7c8..56e1080adab 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkDataExportJobSchedulingHelperImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkDataExportJobSchedulingHelperImplTest.java @@ -30,6 +30,8 @@ import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import jakarta.annotation.Nonnull; + +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; @@ -43,6 +45,7 @@ import java.util.stream.IntStream; import static org.exparity.hamcrest.date.DateMatchers.within; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -97,7 +100,17 @@ public class BulkDataExportJobSchedulingHelperImplTest { verify(myJpaJobPersistence, never()).deleteInstanceAndChunks(anyString()); final Date cutoffDate = myCutoffCaptor.getValue(); - assertEquals(DateUtils.truncate(computeDateFromConfig(expectedRetentionHours), Calendar.SECOND), DateUtils.truncate(cutoffDate, Calendar.SECOND)); + Date expectedCutoff = computeDateFromConfig(expectedRetentionHours); + verifyDatesWithinSeconds(expectedCutoff, cutoffDate, 2); + } + + private void verifyDatesWithinSeconds(Date theExpected, Date theActual, int theSeconds) { + Instant expectedInstant = theExpected.toInstant(); + Instant actualInstant = theActual.toInstant(); + + String msg = String.format("Expected time not within %d s", theSeconds); + assertTrue(expectedInstant.plus(theSeconds, ChronoUnit.SECONDS).isAfter(actualInstant), msg); + assertTrue(expectedInstant.minus(theSeconds, ChronoUnit.SECONDS).isBefore(actualInstant), msg); } @Test diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml b/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml index bb2281e78d6..157c68d5c0c 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-hfql/pom.xml b/hapi-fhir-jpaserver-hfql/pom.xml index 1bdf1f4d5eb..706d5ca1335 100644 --- a/hapi-fhir-jpaserver-hfql/pom.xml +++ b/hapi-fhir-jpaserver-hfql/pom.xml @@ -3,7 +3,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-ips/pom.xml b/hapi-fhir-jpaserver-ips/pom.xml index 164b0a28831..ca914b4b1bd 100644 --- a/hapi-fhir-jpaserver-ips/pom.xml +++ b/hapi-fhir-jpaserver-ips/pom.xml @@ -3,7 +3,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/pom.xml b/hapi-fhir-jpaserver-mdm/pom.xml index 1a7598268ab..3030cb455b6 100644 --- a/hapi-fhir-jpaserver-mdm/pom.xml +++ b/hapi-fhir-jpaserver-mdm/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index 2cee62cefaf..d2028f332ae 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BaseTag.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BaseTag.java index 54d2405fba3..0bf655a77ae 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BaseTag.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/BaseTag.java @@ -31,6 +31,7 @@ public abstract class BaseTag extends BasePartitionable implements Serializable private static final long serialVersionUID = 1L; + // many baseTags -> one tag definition @ManyToOne(cascade = {}) @JoinColumn(name = "TAG_ID", nullable = false) private TagDefinition myTag; diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/TagDefinition.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/TagDefinition.java index fe3868c8313..ad4e6309508 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/TagDefinition.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/TagDefinition.java @@ -67,12 +67,14 @@ public class TagDefinition implements Serializable { @Column(name = "TAG_ID") private Long myId; + // one tag definition -> many resource tags @OneToMany( cascade = {}, fetch = FetchType.LAZY, mappedBy = "myTag") private Collection myResources; + // one tag definition -> many history @OneToMany( cascade = {}, fetch = FetchType.LAZY, diff --git a/hapi-fhir-jpaserver-searchparam/pom.xml b/hapi-fhir-jpaserver-searchparam/pom.xml index f0e7c3a6b94..87f9dd268b2 100755 --- a/hapi-fhir-jpaserver-searchparam/pom.xml +++ b/hapi-fhir-jpaserver-searchparam/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-subscription/pom.xml b/hapi-fhir-jpaserver-subscription/pom.xml index a79817e0242..e608341ec73 100644 --- a/hapi-fhir-jpaserver-subscription/pom.xml +++ b/hapi-fhir-jpaserver-subscription/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-subscription/src/test/java/ca/uhn/fhir/jpa/subscription/module/subscriber/SubscriptionMatchingSubscriberTest.java b/hapi-fhir-jpaserver-subscription/src/test/java/ca/uhn/fhir/jpa/subscription/module/subscriber/SubscriptionMatchingSubscriberTest.java index 1c0dcd424e5..3581da0a156 100644 --- a/hapi-fhir-jpaserver-subscription/src/test/java/ca/uhn/fhir/jpa/subscription/module/subscriber/SubscriptionMatchingSubscriberTest.java +++ b/hapi-fhir-jpaserver-subscription/src/test/java/ca/uhn/fhir/jpa/subscription/module/subscriber/SubscriptionMatchingSubscriberTest.java @@ -505,7 +505,7 @@ public class SubscriptionMatchingSubscriberTest extends BaseBlockingQueueSubscri subscriber.matchActiveSubscriptionsAndDeliver(message); - verify(myCanonicalSubscription, atLeastOnce()).getSendDeleteMessages(); + verify(myCanonicalSubscription).getSendDeleteMessages(); } @Test diff --git a/hapi-fhir-jpaserver-test-dstu2/pom.xml b/hapi-fhir-jpaserver-test-dstu2/pom.xml index a95f7d93414..ccc242c944d 100644 --- a/hapi-fhir-jpaserver-test-dstu2/pom.xml +++ b/hapi-fhir-jpaserver-test-dstu2/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java index 1d0902574bf..c27ecc6bc2d 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2Test.java @@ -2209,28 +2209,28 @@ public class FhirResourceDaoDstu2Test extends BaseJpaDstu2Test { p.addName().addFamily(methodName); IIdType id1 = myPatientDao.create(p, mySrd).getId().toUnqualifiedVersionless(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); p = new Patient(); p.addIdentifier().setSystem("urn:system2").setValue(methodName); p.addName().addFamily(methodName); IIdType id2 = myPatientDao.create(p, mySrd).getId().toUnqualifiedVersionless(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); p = new Patient(); p.addIdentifier().setSystem("urn:system3").setValue(methodName); p.addName().addFamily(methodName); IIdType id3 = myPatientDao.create(p, mySrd).getId().toUnqualifiedVersionless(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); p = new Patient(); p.addIdentifier().setSystem("urn:system4").setValue(methodName); p.addName().addFamily(methodName); IIdType id4 = myPatientDao.create(p, mySrd).getId().toUnqualifiedVersionless(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); SearchParameterMap pm; List actual; diff --git a/hapi-fhir-jpaserver-test-dstu3/pom.xml b/hapi-fhir-jpaserver-test-dstu3/pom.xml index c94281fbf8f..bb969873766 100644 --- a/hapi-fhir-jpaserver-test-dstu3/pom.xml +++ b/hapi-fhir-jpaserver-test-dstu3/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r4/pom.xml b/hapi-fhir-jpaserver-test-r4/pom.xml index a9b47640828..cc21ea5b56e 100644 --- a/hapi-fhir-jpaserver-test-r4/pom.xml +++ b/hapi-fhir-jpaserver-test-r4/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java index 2e304e08373..f09f823f7a5 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java @@ -10,6 +10,7 @@ import ca.uhn.fhir.batch2.api.IJobStepWorker; import ca.uhn.fhir.batch2.api.ILastJobStepWorker; import ca.uhn.fhir.batch2.api.IReductionStepWorker; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; +import ca.uhn.fhir.batch2.api.RetryChunkLaterException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; @@ -27,15 +28,20 @@ import ca.uhn.fhir.jpa.subscription.channel.api.IChannelFactory; import ca.uhn.fhir.jpa.subscription.channel.impl.LinkedBlockingChannel; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.Batch2JobHelper; +import ca.uhn.fhir.jpa.test.config.Batch2FastSchedulerConfig; +import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; +import ca.uhn.fhir.test.utilities.UnregisterScheduledProcessor; import ca.uhn.fhir.util.JsonUtil; import ca.uhn.test.concurrency.PointcutLatch; +import ca.uhn.test.util.LogbackCaptureTestExtension; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.annotation.Nonnull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; @@ -43,11 +49,21 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Sort; +import org.springframework.messaging.MessageHandler; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -60,6 +76,13 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +@ContextConfiguration(classes = { + Batch2FastSchedulerConfig.class +}) +@TestPropertySource(properties = { + // These tests require scheduling to work + UnregisterScheduledProcessor.SCHEDULING_DISABLED_EQUALS_FALSE +}) public class Batch2CoordinatorIT extends BaseJpaR4Test { private static final Logger ourLog = LoggerFactory.getLogger(Batch2CoordinatorIT.class); @@ -81,6 +104,9 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { @Autowired IJobPersistence myJobPersistence; + @RegisterExtension + LogbackCaptureTestExtension myLogbackCaptureTestExtension = new LogbackCaptureTestExtension(); + private final PointcutLatch myFirstStepLatch = new PointcutLatch("First Step"); private final PointcutLatch myLastStepLatch = new PointcutLatch("Last Step"); private IJobCompletionHandler myCompletionHandler; @@ -91,6 +117,10 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { return RunOutcome.SUCCESS; } + static { + TestR4Config.ourMaxThreads = 100; + } + @Override @BeforeEach public void before() throws Exception { @@ -117,7 +147,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { // final step ILastJobStepWorker last = (step, sink) -> RunOutcome.SUCCESS; // job definition - String jobId = new Exception().getStackTrace()[0].getMethodName(); + String jobId = getMethodNameForJobId(); JobDefinition jd = JobDefinition.newBuilder() .setJobDefinitionId(jobId) .setJobDescription("test job") @@ -183,7 +213,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { IJobStepWorker firstStep = (step, sink) -> callLatch(myFirstStepLatch, step); IJobStepWorker lastStep = (step, sink) -> fail(); - String jobId = new Exception().getStackTrace()[0].getMethodName(); + String jobId = getMethodNameForJobId(); JobDefinition definition = buildGatedJobDefinition(jobId, firstStep, lastStep); myJobDefinitionRegistry.addJobDefinition(definition); @@ -192,6 +222,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { myFirstStepLatch.setExpectedCount(1); Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); + myBatch2JobHelper.runMaintenancePass(); myFirstStepLatch.awaitExpected(); myBatch2JobHelper.awaitJobCompletion(startResponse.getInstanceId()); @@ -216,11 +247,10 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { myFirstStepLatch.setExpectedCount(1); myLastStepLatch.setExpectedCount(1); String batchJobId = myJobCoordinator.startInstance(new SystemRequestDetails(), request).getInstanceId(); + myBatch2JobHelper.runMaintenancePass(); myFirstStepLatch.awaitExpected(); - myBatch2JobHelper.assertFastTracking(batchJobId); - // Since there was only one chunk, the job should proceed without requiring a maintenance pass myBatch2JobHelper.awaitJobCompletion(batchJobId); myLastStepLatch.awaitExpected(); @@ -234,10 +264,92 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { assertEquals(1.0, jobInstance.getProgress()); } + /** + * This test verifies that if we have a workchunks being processed by the queue, + * and the maintenance job kicks in, it won't necessarily advance the steps. + */ @Test - public void reductionStepFailing_willFailJob() throws InterruptedException { + public void gatedJob_whenMaintenanceRunHappensDuringMsgProcessing_doesNotAdvance() throws InterruptedException { // setup - String jobId = new Exception().getStackTrace()[0].getMethodName(); + // we disable the scheduler because multiple schedulers running simultaneously + // might cause database collisions we do not expect (not what we're testing) + myBatch2JobHelper.enableMaintenanceRunner(false); + String jobId = getMethodNameForJobId(); + int chunksToMake = 5; + AtomicInteger secondGateCounter = new AtomicInteger(); + AtomicBoolean reductionCheck = new AtomicBoolean(false); + // we will listen into the message queue so we can force actions on it + MessageHandler handler = message -> { + /* + * We will force a run of the maintenance job + * to simulate the situation in which a chunk is + * still being processed by the WorkChunkMessageHandler + * (and thus, not available yet). + */ + myBatch2JobHelper.forceRunMaintenancePass(); + }; + + buildAndDefine3StepReductionJob(jobId, new IReductionStepHandler() { + + @Override + public void firstStep(StepExecutionDetails theStep, IJobDataSink theDataSink) { + for (int i = 0; i < chunksToMake; i++) { + theDataSink.accept(new FirstStepOutput()); + } + } + + @Override + public void secondStep(StepExecutionDetails theStep, IJobDataSink theDataSink) { + // no new chunks + SecondStepOutput output = new SecondStepOutput(); + theDataSink.accept(output); + } + + @Override + public void reductionStepConsume(ChunkExecutionDetails theChunkDetails, IJobDataSink theDataSink) { + // we expect to get one here + int val = secondGateCounter.getAndIncrement(); + } + + @Override + public void reductionStepRun(StepExecutionDetails theStepExecutionDetails, IJobDataSink theDataSink) { + reductionCheck.set(true); + theDataSink.accept(new ReductionStepOutput(new ArrayList<>())); + } + }); + + try { + myWorkChannel.subscribe(handler); + + // test + JobInstanceStartRequest request = buildRequest(jobId); + myFirstStepLatch.setExpectedCount(1); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); + + String instanceId = startResponse.getInstanceId(); + + // wait + myBatch2JobHelper.awaitJobCompletion(instanceId); + + // verify + Optional instanceOp = myJobPersistence.fetchInstance(instanceId); + assertTrue(instanceOp.isPresent()); + JobInstance jobInstance = instanceOp.get(); + assertTrue(reductionCheck.get()); + assertEquals(chunksToMake, secondGateCounter.get()); + + assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus()); + assertEquals(1.0, jobInstance.getProgress()); + } finally { + myWorkChannel.unsubscribe(handler); + myBatch2JobHelper.enableMaintenanceRunner(true); + } + } + + @Test + public void reductionStepFailing_willFailJob() { + // setup + String jobId = getMethodNameForJobId(); int totalChunks = 3; AtomicInteger chunkCounter = new AtomicInteger(); String error = "this is an error"; @@ -292,22 +404,17 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { @Test public void testJobWithReductionStepFiresCompletionHandler() throws InterruptedException { // setup - String jobId = new Exception().getStackTrace()[0].getMethodName(); + String jobId = getMethodNameForJobId(); String testInfo = "test"; int totalCalls = 2; AtomicInteger secondStepInt = new AtomicInteger(); AtomicBoolean completionBool = new AtomicBoolean(); - AtomicBoolean jobStatusBool = new AtomicBoolean(); - myCompletionHandler = (params) -> { - // ensure our completion handler fires + // ensure our completion handler gets the right status + assertEquals(StatusEnum.COMPLETED, params.getInstance().getStatus()); completionBool.getAndSet(true); - - if (StatusEnum.COMPLETED.equals(params.getInstance().getStatus())){ - jobStatusBool.getAndSet(true); - } }; buildAndDefine3StepReductionJob(jobId, new IReductionStepHandler() { @@ -351,10 +458,11 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); String instanceId = startResponse.getInstanceId(); + myBatch2JobHelper.runMaintenancePass(); myFirstStepLatch.awaitExpected(); assertNotNull(instanceId); - myBatch2JobHelper.awaitGatedStepId(FIRST_STEP_ID, instanceId); + myBatch2JobHelper.awaitGatedStepId(SECOND_STEP_ID, instanceId); // wait for last step to finish ourLog.info("Setting last step latch"); @@ -362,17 +470,16 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { // waiting myBatch2JobHelper.awaitJobCompletion(instanceId); - myLastStepLatch.awaitExpected(); ourLog.info("awaited the last step"); + myLastStepLatch.awaitExpected(); // verify Optional instanceOp = myJobPersistence.fetchInstance(instanceId); assertTrue(instanceOp.isPresent()); JobInstance jobInstance = instanceOp.get(); - // ensure our completion handler fires with the up-to-date job instance + // ensure our completion handler fired assertTrue(completionBool.get()); - assertTrue(jobStatusBool.get()); assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus()); assertEquals(1.0, jobInstance.getProgress()); @@ -382,7 +489,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { @ValueSource(booleans = {true, false}) public void testJobDefinitionWithReductionStepIT(boolean theDelayReductionStepBool) throws InterruptedException { // setup - String jobId = new Exception().getStackTrace()[0].getMethodName() + "_" + theDelayReductionStepBool; + String jobId = getMethodNameForJobId() + "_" + theDelayReductionStepBool; String testInfo = "test"; AtomicInteger secondStepInt = new AtomicInteger(); @@ -441,12 +548,12 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { JobInstanceStartRequest request = buildRequest(jobId); myFirstStepLatch.setExpectedCount(1); Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); - String instanceId = startResponse.getInstanceId(); + myBatch2JobHelper.runMaintenancePass(); myFirstStepLatch.awaitExpected(); assertNotNull(instanceId); - myBatch2JobHelper.awaitGatedStepId(FIRST_STEP_ID, instanceId); + myBatch2JobHelper.awaitGatedStepId(SECOND_STEP_ID, instanceId); // wait for last step to finish ourLog.info("Setting last step latch"); @@ -482,6 +589,95 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { assertEquals(1.0, jobInstance.getProgress()); } + @Test + public void testJobWithLongPollingStep() throws InterruptedException { + // create job definition + int callsToMake = 3; + int chunksToAwait = 2; + String jobId = getMethodNameForJobId(); + + ConcurrentHashMap chunkToCounter = new ConcurrentHashMap<>(); + HashMap chunkToCallsToMake = new HashMap<>(); + IJobStepWorker first = (step, sink) -> { + for (int i = 0; i < chunksToAwait; i++) { + String cv = "chunk" + i; + chunkToCallsToMake.put(cv, callsToMake); + sink.accept(new FirstStepOutput().setValue(cv)); + } + return RunOutcome.SUCCESS; + }; + + // step 2 + IJobStepWorker second = (step, sink) -> { + // simulate a call + Awaitility.await().atMost(100, TimeUnit.MICROSECONDS); + + // we use Batch2FastSchedulerConfig, so we have a fast scheduler + // that should catch and call repeatedly pretty quickly + String chunkValue = step.getData().myTestValue; + AtomicInteger pollCounter = chunkToCounter.computeIfAbsent(chunkValue, (key) -> { + return new AtomicInteger(); + }); + int count = pollCounter.getAndIncrement(); + + if (chunkToCallsToMake.get(chunkValue) <= count) { + sink.accept(new SecondStepOutput()); + return RunOutcome.SUCCESS; + } + throw new RetryChunkLaterException(Duration.of(200, ChronoUnit.MILLIS)); + }; + + // step 3 + ILastJobStepWorker last = (step, sink) -> { + myLastStepLatch.call(1); + return RunOutcome.SUCCESS; + }; + + JobDefinition jd = JobDefinition.newBuilder() + .setJobDefinitionId(jobId) + .setJobDescription("test job") + .setJobDefinitionVersion(TEST_JOB_VERSION) + .setParametersType(TestJobParameters.class) + .gatedExecution() + .addFirstStep( + FIRST_STEP_ID, + "First step", + FirstStepOutput.class, + first + ) + .addIntermediateStep(SECOND_STEP_ID, + "Second step", + SecondStepOutput.class, + second) + .addLastStep( + LAST_STEP_ID, + "Final step", + last + ) + .completionHandler(myCompletionHandler) + .build(); + myJobDefinitionRegistry.addJobDefinition(jd); + + // test + JobInstanceStartRequest request = buildRequest(jobId); + myLastStepLatch.setExpectedCount(chunksToAwait); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); + String instanceId = startResponse.getInstanceId(); + + // waiting for the job + myBatch2JobHelper.awaitJobCompletion(startResponse); + // ensure final step fired + myLastStepLatch.awaitExpected(); + + // verify + assertEquals(chunksToAwait, chunkToCounter.size()); + for (Map.Entry set : chunkToCounter.entrySet()) { + // +1 because after 0 indexing; it will make callsToMake failed calls (0, 1... callsToMake) + // and one more successful call (callsToMake + 1) + assertEquals(callsToMake + 1, set.getValue().get()); + } + } + @Test public void testFirstStepToSecondStep_doubleChunk_doesNotFastTrack() throws InterruptedException { IJobStepWorker firstStep = (step, sink) -> { @@ -491,7 +687,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { }; IJobStepWorker lastStep = (step, sink) -> callLatch(myLastStepLatch, step); - String jobDefId = new Exception().getStackTrace()[0].getMethodName(); + String jobDefId = getMethodNameForJobId(); JobDefinition definition = buildGatedJobDefinition(jobDefId, firstStep, lastStep); myJobDefinitionRegistry.addJobDefinition(definition); @@ -501,6 +697,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { myFirstStepLatch.setExpectedCount(1); Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); String instanceId = startResponse.getInstanceId(); + myBatch2JobHelper.runMaintenancePass(); myFirstStepLatch.awaitExpected(); myLastStepLatch.setExpectedCount(2); @@ -513,14 +710,14 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { @Test - public void JobExecutionFailedException_CausesInstanceFailure() { + public void jobExecutionFailedException_CausesInstanceFailure() { // setup IJobStepWorker firstStep = (step, sink) -> { throw new JobExecutionFailedException("Expected Test Exception"); }; IJobStepWorker lastStep = (step, sink) -> fail(); - String jobDefId = new Exception().getStackTrace()[0].getMethodName(); + String jobDefId = getMethodNameForJobId(); JobDefinition definition = buildGatedJobDefinition(jobDefId, firstStep, lastStep); myJobDefinitionRegistry.addJobDefinition(definition); @@ -538,36 +735,47 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { @Test public void testUnknownException_KeepsInProgress_CanCancelManually() throws InterruptedException { // setup - IJobStepWorker firstStep = (step, sink) -> { - callLatch(myFirstStepLatch, step); - throw new RuntimeException("Expected Test Exception"); - }; - IJobStepWorker lastStep = (step, sink) -> fail(); - String jobDefId = new Exception().getStackTrace()[0].getMethodName(); - JobDefinition definition = buildGatedJobDefinition(jobDefId, firstStep, lastStep); + // we want to control the maintenance runner ourselves in this case + // to prevent intermittent test failures + myJobMaintenanceService.enableMaintenancePass(false); - myJobDefinitionRegistry.addJobDefinition(definition); + try { + IJobStepWorker firstStep = (step, sink) -> { + callLatch(myFirstStepLatch, step); + throw new RuntimeException("Expected Test Exception"); + }; + IJobStepWorker lastStep = (step, sink) -> fail(); - JobInstanceStartRequest request = buildRequest(jobDefId); + String jobDefId = getMethodNameForJobId(); + JobDefinition definition = buildGatedJobDefinition(jobDefId, firstStep, lastStep); - // execute - ourLog.info("Starting job"); - myFirstStepLatch.setExpectedCount(1); - Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); - String instanceId = startResponse.getInstanceId(); - myFirstStepLatch.awaitExpected(); + myJobDefinitionRegistry.addJobDefinition(definition); - // validate - myBatch2JobHelper.awaitJobInProgress(instanceId); + JobInstanceStartRequest request = buildRequest(jobDefId); - // execute - ourLog.info("Cancel job {}", instanceId); - myJobCoordinator.cancelInstance(instanceId); - ourLog.info("Cancel job {} done", instanceId); + // execute + ourLog.info("Starting job"); + myFirstStepLatch.setExpectedCount(1); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), request); + String instanceId = startResponse.getInstanceId(); + myBatch2JobHelper.forceRunMaintenancePass(); + myFirstStepLatch.awaitExpected(); - // validate - myBatch2JobHelper.awaitJobCancelled(instanceId); + // validate + myBatch2JobHelper.awaitJobHasStatusWithForcedMaintenanceRuns(instanceId, StatusEnum.IN_PROGRESS); + + // execute + ourLog.info("Cancel job {}", instanceId); + myJobCoordinator.cancelInstance(instanceId); + ourLog.info("Cancel job {} done", instanceId); + + // validate + myBatch2JobHelper.awaitJobHasStatusWithForcedMaintenanceRuns(instanceId, + StatusEnum.CANCELLED); + } finally { + myJobMaintenanceService.enableMaintenancePass(true); + } } @Test @@ -586,7 +794,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { return RunOutcome.SUCCESS; }; // job definition - String jobDefId = new Exception().getStackTrace()[0].getMethodName(); + String jobDefId = getMethodNameForJobId(); JobDefinition jd = JobDefinition.newBuilder() .setJobDefinitionId(jobDefId) .setJobDescription("test job") @@ -629,6 +837,15 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { return request; } + /** + * Returns the method name of the calling method for a unique job id. + * It is best this is called from the test method directly itself, and never + * delegate to a separate child method.s + */ + private String getMethodNameForJobId() { + return new Exception().getStackTrace()[1].getMethodName(); + } + @Nonnull private JobDefinition buildGatedJobDefinition(String theJobId, IJobStepWorker theFirstStep, IJobStepWorker theLastStep) { return JobDefinition.newBuilder() @@ -723,6 +940,7 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { ) .completionHandler(myCompletionHandler) .build(); + myJobDefinitionRegistry.removeJobDefinition(theJobId, 1); myJobDefinitionRegistry.addJobDefinition(jd); } @@ -732,8 +950,16 @@ public class Batch2CoordinatorIT extends BaseJpaR4Test { } static class FirstStepOutput implements IModelJson { + @JsonProperty("test") + private String myTestValue; + FirstStepOutput() { } + + public FirstStepOutput setValue(String theV) { + myTestValue = theV; + return this; + } } static class SecondStepOutput implements IModelJson { diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java index 1605ada7de8..01e0ab868f9 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java @@ -1,12 +1,10 @@ package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.batch2.model.StatusEnum; -import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Arrays; import java.util.Date; @@ -18,9 +16,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class Batch2JobInstanceRepositoryTest extends BaseJpaR4Test { - @Autowired - IBatch2JobInstanceRepository myBatch2JobInstanceRepository; - @ParameterizedTest @CsvSource({ "QUEUED, FAILED, QUEUED, true, normal transition", @@ -38,16 +33,16 @@ public class Batch2JobInstanceRepositoryTest extends BaseJpaR4Test { entity.setStatus(theCurrentState); entity.setCreateTime(new Date()); entity.setDefinitionId("definition_id"); - myBatch2JobInstanceRepository.save(entity); + myJobInstanceRepository.save(entity); // when int changeCount = runInTransaction(()-> - myBatch2JobInstanceRepository.updateInstanceStatusIfIn(jobId, theTargetState, theAllowedPriorStates)); + myJobInstanceRepository.updateInstanceStatusIfIn(jobId, theTargetState, theAllowedPriorStates)); // then Batch2JobInstanceEntity readBack = runInTransaction(() -> - myBatch2JobInstanceRepository.findById(jobId).orElseThrow()); + myJobInstanceRepository.findById(jobId).orElseThrow()); if (theExpectedSuccessFlag) { assertEquals(1, changeCount, "The change happened"); assertEquals(theTargetState, readBack.getStatus()); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java index 8030e908f5a..f5143b9ffbc 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java @@ -27,6 +27,7 @@ import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.JsonUtil; import ca.uhn.test.concurrency.IPointcutLatch; import ca.uhn.test.concurrency.PointcutLatch; +import jakarta.annotation.Nonnull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -39,7 +40,6 @@ import org.springframework.messaging.MessageChannel; import org.springframework.messaging.support.ChannelInterceptor; import org.springframework.transaction.support.TransactionTemplate; -import jakarta.annotation.Nonnull; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -358,10 +358,10 @@ public class Batch2JobMaintenanceDatabaseIT extends BaseJpaR4Test { WorkChunkExpectation expectation = new WorkChunkExpectation( """ -chunk1, FIRST, COMPLETED -chunk2, SECOND, QUEUED -chunk3, LAST, QUEUED -""", + chunk1, FIRST, COMPLETED + chunk2, SECOND, QUEUED + chunk3, LAST, QUEUED + """, "" ); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java index 6f989437e6e..c39c72f609c 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java @@ -11,17 +11,26 @@ import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; import ca.uhn.fhir.batch2.maintenance.JobMaintenanceServiceImpl; import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.batch2.model.JobWorkNotificationJsonMessage; +import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.jpa.subscription.channel.api.ChannelConsumerSettings; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelFactory; import ca.uhn.fhir.jpa.subscription.channel.impl.LinkedBlockingChannel; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.Batch2JobHelper; +import ca.uhn.fhir.jpa.test.config.Batch2FastSchedulerConfig; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.test.utilities.UnregisterScheduledProcessor; +import ca.uhn.fhir.testjob.TestJobDefinitionUtils; +import ca.uhn.fhir.testjob.models.FirstStepOutput; +import ca.uhn.fhir.testjob.models.ReductionStepOutput; +import ca.uhn.fhir.testjob.models.TestJobParameters; import ca.uhn.test.concurrency.PointcutLatch; import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.annotation.Nonnull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,8 +40,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import jakarta.annotation.Nonnull; -import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; @@ -41,9 +48,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * The on-enter actions are defined in - * {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater#handleStatusChange} + * {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater#handleStatusChange(JobInstance)}} * {@link ca.uhn.fhir.batch2.progress.InstanceProgress#updateStatus(JobInstance)} - * {@link JobInstanceProcessor#cleanupInstance()} + * {@link ca.uhn.fhir.batch2.maintenance.JobInstanceProcessor#cleanupInstance()} * For chunks: * {@link ca.uhn.fhir.jpa.batch2.JpaJobPersistenceImpl#onWorkChunkCreate} @@ -53,13 +60,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @TestPropertySource(properties = { UnregisterScheduledProcessor.SCHEDULING_DISABLED_EQUALS_FALSE }) -@ContextConfiguration(classes = {Batch2JobMaintenanceIT.SpringConfig.class}) +@ContextConfiguration(classes = {Batch2FastSchedulerConfig.class}) public class Batch2JobMaintenanceIT extends BaseJpaR4Test { private static final Logger ourLog = LoggerFactory.getLogger(Batch2JobMaintenanceIT.class); - public static final int TEST_JOB_VERSION = 1; - public static final String FIRST_STEP_ID = "first-step"; - public static final String LAST_STEP_ID = "last-step"; @Autowired JobDefinitionRegistry myJobDefinitionRegistry; @Autowired @@ -87,6 +91,7 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { @BeforeEach public void before() { + myStorageSettings.setJobFastTrackingEnabled(true); myCompletionHandler = details -> {}; myWorkChannel = (LinkedBlockingChannel) myChannelFactory.getOrCreateReceiver(CHANNEL_NAME, JobWorkNotificationJsonMessage.class, new ChannelConsumerSettings()); JobMaintenanceServiceImpl jobMaintenanceService = (JobMaintenanceServiceImpl) myJobMaintenanceService; @@ -99,7 +104,6 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { @AfterEach public void after() { myWorkChannel.clearInterceptorsForUnitTest(); - myStorageSettings.setJobFastTrackingEnabled(true); JobMaintenanceServiceImpl jobMaintenanceService = (JobMaintenanceServiceImpl) myJobMaintenanceService; jobMaintenanceService.setMaintenanceJobStartedCallback(() -> {}); } @@ -122,7 +126,8 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { myFirstStepLatch.setExpectedCount(1); myLastStepLatch.setExpectedCount(1); - String batchJobId = myJobCoordinator.startInstance(request).getInstanceId(); + String batchJobId = myJobCoordinator.startInstance(new SystemRequestDetails(), request).getInstanceId(); + myFirstStepLatch.awaitExpected(); myBatch2JobHelper.assertFastTracking(batchJobId); @@ -156,12 +161,12 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { public void testFirstStepToSecondStepFasttrackingDisabled_singleChunkDoesNotFasttrack() throws InterruptedException { myStorageSettings.setJobFastTrackingEnabled(false); - IJobStepWorker firstStep = (step, sink) -> { - sink.accept(new Batch2JobMaintenanceIT.FirstStepOutput()); + IJobStepWorker firstStep = (step, sink) -> { + sink.accept(new FirstStepOutput()); callLatch(myFirstStepLatch, step); return RunOutcome.SUCCESS; }; - IJobStepWorker lastStep = (step, sink) -> callLatch(myLastStepLatch, step); + IJobStepWorker lastStep = (step, sink) -> callLatch(myLastStepLatch, step); String jobDefId = new Exception().getStackTrace()[0].getMethodName(); @@ -173,7 +178,7 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { myFirstStepLatch.setExpectedCount(1); myLastStepLatch.setExpectedCount(1); - String batchJobId = myJobCoordinator.startInstance(request).getInstanceId(); + String batchJobId = myJobCoordinator.startInstance(new SystemRequestDetails(), request).getInstanceId(); myFirstStepLatch.awaitExpected(); myBatch2JobHelper.assertFastTracking(batchJobId); @@ -200,65 +205,20 @@ public class Batch2JobMaintenanceIT extends BaseJpaR4Test { @Nonnull private JobDefinition buildGatedJobDefinition(String theJobId, IJobStepWorker theFirstStep, IJobStepWorker theLastStep) { - return JobDefinition.newBuilder() - .setJobDefinitionId(theJobId) - .setJobDescription("test job") - .setJobDefinitionVersion(TEST_JOB_VERSION) - .setParametersType(TestJobParameters.class) - .gatedExecution() - .addFirstStep( - FIRST_STEP_ID, - "Test first step", - FirstStepOutput.class, - theFirstStep - ) - .addLastStep( - LAST_STEP_ID, - "Test last step", - theLastStep - ) - .completionHandler(myCompletionHandler) - .build(); + return TestJobDefinitionUtils.buildGatedJobDefinition( + theJobId, + theFirstStep, + theLastStep, + myCompletionHandler + ); } - static class TestJobParameters implements IModelJson { - TestJobParameters() { - } - } - - static class FirstStepOutput implements IModelJson { - FirstStepOutput() { - } - } - - static class SecondStepOutput implements IModelJson { - @JsonProperty("test") - private String myTestValue; - - SecondStepOutput() { - } - - public void setValue(String theV) { - myTestValue = theV; - } - } - - static class ReductionStepOutput implements IModelJson { + static class OurReductionStepOutput extends ReductionStepOutput { @JsonProperty("result") private List myResult; - ReductionStepOutput(List theResult) { + OurReductionStepOutput(List theResult) { myResult = theResult; } } - - static class SpringConfig { - @Autowired - IJobMaintenanceService myJobMaintenanceService; - - @PostConstruct - void fastScheduler() { - ((JobMaintenanceServiceImpl)myJobMaintenanceService).setScheduledJobFrequencyMillis(200); - } - } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java index b8b16b4670f..e4fded6cc3e 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java @@ -1,7 +1,6 @@ package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.batch2.api.IJobCoordinator; -import ca.uhn.fhir.rest.api.server.bulk.BulkExportJobParameters; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; @@ -10,10 +9,13 @@ import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.server.bulk.BulkExportJobParameters; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.util.Batch2JobDefinitionConstants; import ca.uhn.fhir.util.JsonUtil; import com.google.common.collect.Sets; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Binary; import org.hl7.fhir.r4.model.Enumerations; import org.hl7.fhir.r4.model.Group; @@ -36,13 +38,16 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.equalTo; @@ -64,6 +69,7 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { @BeforeEach void beforeEach() { + ourLog.info("BulkDataErrorAbuseTest.beforeEach"); afterPurgeDatabase(); } @@ -93,7 +99,7 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { duAbuseTest(Integer.MAX_VALUE); } - private void duAbuseTest(int taskExecutions) throws InterruptedException, ExecutionException { + private void duAbuseTest(int taskExecutions) { // Create some resources Patient patient = new Patient(); patient.setId("PING1"); @@ -133,18 +139,19 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { ExecutorService executorService = new ThreadPoolExecutor(workerCount, workerCount, 0L, TimeUnit.MILLISECONDS, workQueue); + CompletionService completionService = new ExecutorCompletionService<>(executorService); ourLog.info("Starting task creation"); - List> futures = new ArrayList<>(); + int maxFuturesToProcess = 500; for (int i = 0; i < taskExecutions; i++) { - futures.add(executorService.submit(() -> { + completionService.submit(() -> { String instanceId = null; try { instanceId = startJob(options); // Run a scheduled pass to build the export - myBatch2JobHelper.awaitJobCompletion(instanceId, 60); + myBatch2JobHelper.awaitJobCompletion(instanceId, 10); verifyBulkExportResults(instanceId, List.of("Patient/PING1", "Patient/PING2"), Collections.singletonList("Patient/PNING3")); @@ -153,14 +160,11 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { ourLog.error("Caught an error during processing instance {}", instanceId, theError); throw new InternalErrorException("Caught an error during processing instance " + instanceId, theError); } - })); + }); // Don't let the list of futures grow so big we run out of memory - if (futures.size() > 1000) { - while (futures.size() > 500) { - // This should always return true, but it'll throw an exception if we failed - assertTrue(futures.remove(0).get()); - } + if (i != 0 && i % maxFuturesToProcess == 0) { + executeFutures(completionService, maxFuturesToProcess); } } @@ -168,18 +172,53 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { // wait for completion to avoid stranding background tasks. executorService.shutdown(); - assertTrue(executorService.awaitTermination(60, TimeUnit.SECONDS), "Finished before timeout"); + await() + .atMost(60, TimeUnit.SECONDS) + .until(() -> { + return executorService.isTerminated() && executorService.isShutdown(); + }); // verify that all requests succeeded ourLog.info("All tasks complete. Verify results."); - for (var next : futures) { - // This should always return true, but it'll throw an exception if we failed - assertTrue(next.get()); - } + executeFutures(completionService, taskExecutions % maxFuturesToProcess); + + executorService.shutdown(); + await() + .atMost(60, TimeUnit.SECONDS) + .until(() -> { + return executorService.isTerminated() && executorService.isShutdown(); + }); ourLog.info("Finished task execution"); } + private void executeFutures(CompletionService theCompletionService, int theTotal) { + List errors = new ArrayList<>(); + int count = 0; + + while (count + errors.size() < theTotal) { + try { + Future future = theCompletionService.take(); + boolean r = future.get(); + assertTrue(r); + count++; + } catch (Exception ex) { + // we will run all the threads to completion, even if we have errors; + // this is so we don't have background threads kicking around with + // partial changes. + // we either do this, or shutdown the completion service in an + // "inelegant" manner, dropping all threads (which we aren't doing) + ourLog.error("Failed after checking " + count + " futures"); + errors.add(ex.getMessage()); + } + } + + if (!errors.isEmpty()) { + fail(String.format("Failed to execute futures. Found %d errors :\n", errors.size()) + + String.join(", ", errors)); + } + } + private void verifyBulkExportResults(String theInstanceId, List theContainedList, List theExcludedList) { // Iterate over the files @@ -196,7 +235,6 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { String resourceType = file.getKey(); List binaryIds = file.getValue(); for (var nextBinaryId : binaryIds) { - Binary binary = myBinaryDao.read(new IdType(nextBinaryId), mySrd); assertEquals(Constants.CT_FHIR_NDJSON, binary.getContentType()); @@ -207,18 +245,17 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { .lines().toList(); ourLog.debug("Export job {} file {} line-count: {}", theInstanceId, nextBinaryId, lines.size()); - lines.stream() - .map(line -> myFhirContext.newJsonParser().parseResource(line)) - .map(r -> r.getIdElement().toUnqualifiedVersionless()) - .forEach(nextId -> { - if (!resourceType.equals(nextId.getResourceType())) { - fail("Found resource of type " + nextId.getResourceType() + " in file for type " + resourceType); - } else { - if (!foundIds.add(nextId.getValue())) { - fail("Found duplicate ID: " + nextId.getValue()); - } + for (String line : lines) { + IBaseResource resource = myFhirContext.newJsonParser().parseResource(line); + IIdType nextId = resource.getIdElement().toUnqualifiedVersionless(); + if (!resourceType.equals(nextId.getResourceType())) { + fail("Found resource of type " + nextId.getResourceType() + " in file for type " + resourceType); + } else { + if (!foundIds.add(nextId.getValue())) { + fail("Found duplicate ID: " + nextId.getValue()); } - }); + } + } } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceRepositoryTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceRepositoryTest.java index 2485daacc91..cd95d6faf27 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceRepositoryTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceRepositoryTest.java @@ -4,7 +4,6 @@ import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; -import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import org.junit.jupiter.api.AfterEach; @@ -23,8 +22,6 @@ import static org.hamcrest.Matchers.hasSize; public class JobInstanceRepositoryTest extends BaseJpaR4Test { - @Autowired - private IBatch2JobInstanceRepository myJobInstanceRepository; @Autowired private IJobPersistence myJobPersistenceSvc; private static final String PARAMS = "{\"param1\":\"value1\"}"; diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java index 90654ff9bc1..fa492384b37 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java @@ -1,9 +1,15 @@ package ca.uhn.fhir.jpa.batch2; +import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobOperationResultJson; +import ca.uhn.fhir.batch2.api.RunOutcome; +import ca.uhn.fhir.batch2.channel.BatchJobSender; +import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; import ca.uhn.fhir.batch2.jobs.imprt.NdJsonFileJson; +import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; @@ -18,26 +24,34 @@ import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.jpa.test.Batch2JobHelper; +import ca.uhn.fhir.jpa.test.config.Batch2FastSchedulerConfig; +import ca.uhn.fhir.testjob.TestJobDefinitionUtils; +import ca.uhn.fhir.testjob.models.FirstStepOutput; import ca.uhn.fhir.util.JsonUtil; import ca.uhn.hapi.fhir.batch2.test.AbstractIJobPersistenceSpecificationTest; import ca.uhn.hapi.fhir.batch2.test.configs.SpyOverrideConfig; +import ca.uhn.test.concurrency.PointcutLatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; +import jakarta.annotation.Nonnull; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; +import org.springframework.test.context.ContextConfiguration; import org.springframework.transaction.PlatformTransactionManager; -import jakarta.annotation.Nonnull; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -60,15 +74,25 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @TestMethodOrder(MethodOrderer.MethodName.class) +@ContextConfiguration(classes = { + Batch2FastSchedulerConfig.class +}) @Import(SpyOverrideConfig.class) public class JpaJobPersistenceImplTest extends BaseJpaR4Test { public static final String JOB_DEFINITION_ID = "definition-id"; - public static final String TARGET_STEP_ID = "step-id"; + public static final String FIRST_STEP_ID = TestJobDefinitionUtils.FIRST_STEP_ID; + public static final String LAST_STEP_ID = TestJobDefinitionUtils.LAST_STEP_ID; public static final String DEF_CHUNK_ID = "definition-chunkId"; - public static final String STEP_CHUNK_ID = "step-chunkId"; + public static final String STEP_CHUNK_ID = TestJobDefinitionUtils.FIRST_STEP_ID; public static final int JOB_DEF_VER = 1; public static final int SEQUENCE_NUMBER = 1; public static final String CHUNK_DATA = "{\"key\":\"value\"}"; @@ -80,6 +104,25 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { @Autowired private IBatch2JobInstanceRepository myJobInstanceRepository; + @Autowired + public Batch2JobHelper myBatch2JobHelper; + + // this is our spy + @Autowired + private BatchJobSender myBatchSender; + + @Autowired + private IJobMaintenanceService myMaintenanceService; + + @Autowired + public JobDefinitionRegistry myJobDefinitionRegistry; + + @AfterEach + public void after() { + myJobDefinitionRegistry.removeJobDefinition(JOB_DEFINITION_ID, JOB_DEF_VER); + myMaintenanceService.enableMaintenancePass(true); + } + @Test public void testDeleteInstance() { // Setup @@ -87,7 +130,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); for (int i = 0; i < 10; i++) { - storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, i, JsonUtil.serialize(new NdJsonFileJson().setNdJsonText("{}"))); + storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, i, JsonUtil.serialize(new NdJsonFileJson().setNdJsonText("{}")), false); } // Execute @@ -102,8 +145,13 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { }); } - private String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData) { - WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData); + private String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData, boolean theGatedExecution) { + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, TestJobDefinitionUtils.TEST_JOB_VERSION, theTargetStepId, theInstanceId, theSequence, theSerializedData, theGatedExecution); + return mySvc.onWorkChunkCreate(batchWorkChunk); + } + + private String storeFirstWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData) { + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, TestJobDefinitionUtils.TEST_JOB_VERSION, theTargetStepId, theInstanceId, theSequence, theSerializedData, false); return mySvc.onWorkChunkCreate(batchWorkChunk); } @@ -113,7 +161,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { String instanceId = mySvc.storeNewInstance(instance); runInTransaction(() -> { - Batch2JobInstanceEntity instanceEntity = myJobInstanceRepository.findById(instanceId).orElseThrow(IllegalStateException::new); + Batch2JobInstanceEntity instanceEntity = findInstanceByIdOrThrow(instanceId); assertEquals(StatusEnum.QUEUED, instanceEntity.getStatus()); }); @@ -126,7 +174,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertEquals(instance.getReport(), foundInstance.getReport()); runInTransaction(() -> { - Batch2JobInstanceEntity instanceEntity = myJobInstanceRepository.findById(instanceId).orElseThrow(IllegalStateException::new); + Batch2JobInstanceEntity instanceEntity = findInstanceByIdOrThrow(instanceId); assertEquals(StatusEnum.QUEUED, instanceEntity.getStatus()); }); } @@ -213,12 +261,14 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { @ParameterizedTest @MethodSource("provideStatuses") - public void testStartChunkOnlyWorksOnValidChunks(WorkChunkStatusEnum theStatus, boolean theShouldBeStartedByConsumer) { + public void testStartChunkOnlyWorksOnValidChunks(WorkChunkStatusEnum theStatus, boolean theShouldBeStartedByConsumer) throws InterruptedException { // Setup JobInstance instance = createInstance(); + myMaintenanceService.enableMaintenancePass(false); String instanceId = mySvc.storeNewInstance(instance); - storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); - WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(JOB_DEFINITION_ID, JOB_DEF_VER, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); + + storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, CHUNK_DATA, false); + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(JOB_DEFINITION_ID, JOB_DEF_VER, FIRST_STEP_ID, instanceId, 0, CHUNK_DATA, false); String chunkId = mySvc.onWorkChunkCreate(batchWorkChunk); Optional byId = myWorkChunkRepository.findById(chunkId); Batch2WorkChunkEntity entity = byId.get(); @@ -230,7 +280,9 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { // Verify boolean chunkStarted = workChunk.isPresent(); - assertEquals(chunkStarted, theShouldBeStartedByConsumer); + assertEquals(theShouldBeStartedByConsumer, chunkStarted); + verify(myBatchSender, never()) + .sendWorkChannelMessage(any()); } @Test @@ -344,46 +396,185 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { @Test public void testUpdateTime() { // Setup - JobInstance instance = createInstance(); + boolean isGatedExecution = false; + JobInstance instance = createInstance(true, isGatedExecution); String instanceId = mySvc.storeNewInstance(instance); - Date updateTime = runInTransaction(() -> new Date(myJobInstanceRepository.findById(instanceId).orElseThrow().getUpdateTime().getTime())); + Date updateTime = runInTransaction(() -> new Date(findInstanceByIdOrThrow(instanceId).getUpdateTime().getTime())); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); // Test runInTransaction(() -> mySvc.updateInstanceUpdateTime(instanceId)); // Verify - Date updateTime2 = runInTransaction(() -> new Date(myJobInstanceRepository.findById(instanceId).orElseThrow().getUpdateTime().getTime())); + Date updateTime2 = runInTransaction(() -> new Date(findInstanceByIdOrThrow(instanceId).getUpdateTime().getTime())); assertNotEquals(updateTime, updateTime2); } + @Test + public void advanceJobStepAndUpdateChunkStatus_forGatedJobWithoutReduction_updatesCurrentStepAndChunkStatus() { + // setup + boolean isGatedExecution = true; + JobInstance instance = createInstance(true, isGatedExecution); + String instanceId = mySvc.storeNewInstance(instance); + String chunkIdSecondStep1 = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, null, isGatedExecution); + String chunkIdSecondStep2 = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, null, isGatedExecution); + + runInTransaction(() -> assertEquals(FIRST_STEP_ID, findInstanceByIdOrThrow(instanceId).getCurrentGatedStepId())); + + // execute + runInTransaction(() -> { + boolean changed = mySvc.advanceJobStepAndUpdateChunkStatus(instanceId, LAST_STEP_ID, false); + assertTrue(changed); + }); + + // verify + runInTransaction(() -> { + assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(chunkIdSecondStep1).getStatus()); + assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(chunkIdSecondStep2).getStatus()); + assertEquals(LAST_STEP_ID, findInstanceByIdOrThrow(instanceId).getCurrentGatedStepId()); + }); + } + + @Test + public void advanceJobStepAndUpdateChunkStatus_whenAlreadyInTargetStep_DoesNotUpdateStepOrChunks() { + // setup + boolean isGatedExecution = true; + JobInstance instance = createInstance(true, isGatedExecution); + String instanceId = mySvc.storeNewInstance(instance); + String chunkIdSecondStep1 = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, null, isGatedExecution); + String chunkIdSecondStep2 = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, null, isGatedExecution); + + runInTransaction(() -> assertEquals(FIRST_STEP_ID, findInstanceByIdOrThrow(instanceId).getCurrentGatedStepId())); + + // execute + runInTransaction(() -> { + boolean changed = mySvc.advanceJobStepAndUpdateChunkStatus(instanceId, FIRST_STEP_ID, false); + assertFalse(changed); + }); + + // verify + runInTransaction(() -> { + assertEquals(WorkChunkStatusEnum.GATE_WAITING, findChunkByIdOrThrow(chunkIdSecondStep1).getStatus()); + assertEquals(WorkChunkStatusEnum.GATE_WAITING, findChunkByIdOrThrow(chunkIdSecondStep2).getStatus()); + assertEquals(FIRST_STEP_ID, findInstanceByIdOrThrow(instanceId).getCurrentGatedStepId()); + }); + } + @Test public void testFetchUnknownWork() { assertFalse(myWorkChunkRepository.findById("FOO").isPresent()); } - @Test - public void testStoreAndFetchWorkChunk_NoData() { - JobInstance instance = createInstance(); + @ParameterizedTest + @CsvSource({ + "false, READY, QUEUED", + "true, GATE_WAITING, QUEUED" + }) + public void testStoreAndFetchWorkChunk_withOrWithoutGatedExecutionNoData_createdAndTransitionToExpectedStatus(boolean theGatedExecution, WorkChunkStatusEnum theExpectedStatusOnCreate, WorkChunkStatusEnum theExpectedStatusAfterTransition) throws InterruptedException { + // setup + JobInstance instance = createInstance(true, theGatedExecution); + + // when + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(1); + myMaintenanceService.enableMaintenancePass(false); String instanceId = mySvc.storeNewInstance(instance); - String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, null); + // execute & verify + String firstChunkId = storeFirstWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, null); + // mark the first chunk as COMPLETED to allow step advance + runInTransaction(() -> myWorkChunkRepository.updateChunkStatus(firstChunkId, WorkChunkStatusEnum.READY, WorkChunkStatusEnum.COMPLETED)); + + String id = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, null, theGatedExecution); + runInTransaction(() -> assertEquals(theExpectedStatusOnCreate, findChunkByIdOrThrow(id).getStatus())); + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> assertEquals(theExpectedStatusAfterTransition, findChunkByIdOrThrow(id).getStatus())); WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); + // assert null since we did not input any data when creating the chunks assertNull(chunk.getData()); + + latch.awaitExpected(); + verify(myBatchSender).sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); + } + + @Test + public void testStoreAndFetchWorkChunk_withGatedJobMultipleChunk_correctTransitions() throws InterruptedException { + // setup + boolean isGatedExecution = true; + String expectedFirstChunkData = "IAmChunk1"; + String expectedSecondChunkData = "IAmChunk2"; + JobInstance instance = createInstance(true, isGatedExecution); + myMaintenanceService.enableMaintenancePass(false); + String instanceId = mySvc.storeNewInstance(instance); + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(2); + + // execute & verify + String firstChunkId = storeFirstWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, expectedFirstChunkData); + String secondChunkId = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, expectedSecondChunkData, isGatedExecution); + + runInTransaction(() -> { + // check chunks created in expected states + assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(firstChunkId).getStatus()); + assertEquals(WorkChunkStatusEnum.GATE_WAITING, findChunkByIdOrThrow(secondChunkId).getStatus()); + }); + + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> { + assertEquals(WorkChunkStatusEnum.QUEUED, findChunkByIdOrThrow(firstChunkId).getStatus()); + // maintenance should not affect chunks in step 2 + assertEquals(WorkChunkStatusEnum.GATE_WAITING, findChunkByIdOrThrow(secondChunkId).getStatus()); + }); + + WorkChunk actualFirstChunkData = mySvc.onWorkChunkDequeue(firstChunkId).orElseThrow(IllegalArgumentException::new); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, findChunkByIdOrThrow(firstChunkId).getStatus())); + assertEquals(expectedFirstChunkData, actualFirstChunkData.getData()); + + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(firstChunkId, 50, 0)); + runInTransaction(() -> { + assertEquals(WorkChunkStatusEnum.COMPLETED, findChunkByIdOrThrow(firstChunkId).getStatus()); + assertEquals(WorkChunkStatusEnum.GATE_WAITING, findChunkByIdOrThrow(secondChunkId).getStatus()); + }); + + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> { + assertEquals(WorkChunkStatusEnum.COMPLETED, findChunkByIdOrThrow(firstChunkId).getStatus()); + // now that all chunks for step 1 is COMPLETED, should enqueue chunks in step 2 + assertEquals(WorkChunkStatusEnum.QUEUED, findChunkByIdOrThrow(secondChunkId).getStatus()); + }); + + WorkChunk actualSecondChunkData = mySvc.onWorkChunkDequeue(secondChunkId).orElseThrow(IllegalArgumentException::new); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, findChunkByIdOrThrow(secondChunkId).getStatus())); + assertEquals(expectedSecondChunkData, actualSecondChunkData.getData()); + + latch.awaitExpected(); + verify(myBatchSender, times(2)) + .sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); } @Test void testStoreAndFetchChunksForInstance_NoData() { // given + boolean isGatedExecution = false; JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); - String queuedId = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, "some data"); - String erroredId = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 1, "some more data"); - String completedId = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 2, "some more data"); + String queuedId = storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, "some data", isGatedExecution); + String erroredId = storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 1, "some more data", isGatedExecution); + String completedId = storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 2, "some more data", isGatedExecution); mySvc.onWorkChunkDequeue(erroredId); WorkChunkErrorEvent parameters = new WorkChunkErrorEvent(erroredId, "Our error message"); @@ -407,9 +598,9 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertEquals(JOB_DEFINITION_ID, workChunk.getJobDefinitionId()); assertEquals(JOB_DEF_VER, workChunk.getJobDefinitionVersion()); assertEquals(instanceId, workChunk.getInstanceId()); - assertEquals(TARGET_STEP_ID, workChunk.getTargetStepId()); + assertEquals(FIRST_STEP_ID, workChunk.getTargetStepId()); assertEquals(0, workChunk.getSequence()); - assertEquals(WorkChunkStatusEnum.QUEUED, workChunk.getStatus()); + assertEquals(WorkChunkStatusEnum.READY, workChunk.getStatus()); assertNotNull(workChunk.getCreateTime()); @@ -418,7 +609,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertNull(workChunk.getEndTime()); assertNull(workChunk.getErrorMessage()); assertEquals(0, workChunk.getErrorCount()); - assertEquals(null, workChunk.getRecordsProcessed()); + assertNull(workChunk.getRecordsProcessed()); } { @@ -426,7 +617,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertEquals(WorkChunkStatusEnum.ERRORED, workChunk1.getStatus()); assertEquals("Our error message", workChunk1.getErrorMessage()); assertEquals(1, workChunk1.getErrorCount()); - assertEquals(null, workChunk1.getRecordsProcessed()); + assertNull(workChunk1.getRecordsProcessed()); assertNotNull(workChunk1.getEndTime()); } @@ -438,18 +629,35 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertNull(workChunk2.getErrorMessage()); assertEquals(0, workChunk2.getErrorCount()); } - } - - @Test - public void testStoreAndFetchWorkChunk_WithData() { - JobInstance instance = createInstance(); + @ParameterizedTest + @CsvSource({ + "false, READY, QUEUED", + "true, GATE_WAITING, QUEUED" + }) + public void testStoreAndFetchWorkChunk_withOrWithoutGatedExecutionwithData_createdAndTransitionToExpectedStatus(boolean theGatedExecution, WorkChunkStatusEnum theExpectedCreatedStatus, WorkChunkStatusEnum theExpectedTransitionStatus) throws InterruptedException { + // setup + JobInstance instance = createInstance(true, theGatedExecution); + myMaintenanceService.enableMaintenancePass(false); String instanceId = mySvc.storeNewInstance(instance); + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(1); - String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); + // execute & verify + String firstChunkId = storeFirstWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, null); + // mark the first chunk as COMPLETED to allow step advance + runInTransaction(() -> myWorkChunkRepository.updateChunkStatus(firstChunkId, WorkChunkStatusEnum.READY, WorkChunkStatusEnum.COMPLETED)); + + String id = storeWorkChunk(JOB_DEFINITION_ID, LAST_STEP_ID, instanceId, 0, CHUNK_DATA, theGatedExecution); assertNotNull(id); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(theExpectedCreatedStatus, findChunkByIdOrThrow(id).getStatus())); + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> assertEquals(theExpectedTransitionStatus, findChunkByIdOrThrow(id).getStatus())); WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); assertEquals(36, chunk.getInstanceId().length()); @@ -458,19 +666,30 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); assertEquals(CHUNK_DATA, chunk.getData()); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, findChunkByIdOrThrow(id).getStatus())); + latch.awaitExpected(); + verify(myBatchSender).sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); } @Test - public void testMarkChunkAsCompleted_Success() { - JobInstance instance = createInstance(); + public void testMarkChunkAsCompleted_Success() throws InterruptedException { + boolean isGatedExecution = false; + myMaintenanceService.enableMaintenancePass(false); + JobInstance instance = createInstance(true, isGatedExecution); String instanceId = mySvc.storeNewInstance(instance); - String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, CHUNK_DATA); + String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, CHUNK_DATA, isGatedExecution); assertNotNull(chunkId); + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(1); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); - - sleepUntilTimeChanges(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(chunkId).getStatus())); + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, findChunkByIdOrThrow(chunkId).getStatus())); WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); @@ -480,13 +699,13 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertNull(chunk.getEndTime()); assertNull(chunk.getRecordsProcessed()); assertNotNull(chunk.getData()); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, findChunkByIdOrThrow(chunkId).getStatus())); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 50, 0)); runInTransaction(() -> { - Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); + Batch2WorkChunkEntity entity = findChunkByIdOrThrow(chunkId); assertEquals(WorkChunkStatusEnum.COMPLETED, entity.getStatus()); assertEquals(50, entity.getRecordsProcessed()); assertNotNull(entity.getCreateTime()); @@ -496,63 +715,41 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); }); - } - - @Test - public void testGatedAdvancementByStatus() { - // Setup - JobInstance instance = createInstance(); - String instanceId = mySvc.storeNewInstance(instance); - String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 0, 0)); - - boolean canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); - assertTrue(canAdvance); - - //Storing a new chunk with QUEUED should prevent advancement. - String newChunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - - canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); - assertFalse(canAdvance); - - //Toggle it to complete - mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(newChunkId, 50, 0)); - canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); - assertTrue(canAdvance); - - //Create a new chunk and set it in progress. - String newerChunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - mySvc.onWorkChunkDequeue(newerChunkId); - canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); - assertFalse(canAdvance); - - //Toggle IN_PROGRESS to complete - mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(newerChunkId, 50, 0)); - canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); - assertTrue(canAdvance); + latch.awaitExpected(); + verify(myBatchSender).sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); } @Test public void testMarkChunkAsCompleted_Error() { - JobInstance instance = createInstance(); + boolean isGatedExecution = false; + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(1); + myMaintenanceService.enableMaintenancePass(false); + + JobInstance instance = createInstance(true, isGatedExecution); String instanceId = mySvc.storeNewInstance(instance); - String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); + String chunkId = storeWorkChunk(JOB_DEFINITION_ID, TestJobDefinitionUtils.FIRST_STEP_ID, instanceId, SEQUENCE_NUMBER, null, isGatedExecution); assertNotNull(chunkId); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); - - sleepUntilTimeChanges(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(chunkId).getStatus())); + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, findChunkByIdOrThrow(chunkId).getStatus())); WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); WorkChunkErrorEvent request = new WorkChunkErrorEvent(chunkId).setErrorMsg("This is an error message"); mySvc.onWorkChunkError(request); runInTransaction(() -> { - Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); + Batch2WorkChunkEntity entity = findChunkByIdOrThrow(chunkId); assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); assertEquals("This is an error message", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); @@ -568,7 +765,7 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { WorkChunkErrorEvent request2 = new WorkChunkErrorEvent(chunkId).setErrorMsg("This is an error message 2"); mySvc.onWorkChunkError(request2); runInTransaction(() -> { - Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); + Batch2WorkChunkEntity entity = findChunkByIdOrThrow(chunkId); assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); assertEquals("This is an error message 2", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); @@ -582,28 +779,39 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { List chunks = ImmutableList.copyOf(mySvc.fetchAllWorkChunksIterator(instanceId, true)); assertEquals(1, chunks.size()); assertEquals(2, chunks.get(0).getErrorCount()); + + verify(myBatchSender).sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); } @Test - public void testMarkChunkAsCompleted_Fail() { - JobInstance instance = createInstance(); + public void testMarkChunkAsCompleted_Fail() throws InterruptedException { + boolean isGatedExecution = false; + myMaintenanceService.enableMaintenancePass(false); + JobInstance instance = createInstance(true, isGatedExecution); String instanceId = mySvc.storeNewInstance(instance); - String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); + String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null, isGatedExecution); assertNotNull(chunkId); + PointcutLatch latch = new PointcutLatch("senderlatch"); + doAnswer(a -> { + latch.call(1); + return Void.class; + }).when(myBatchSender).sendWorkChannelMessage(any(JobWorkNotification.class)); + latch.setExpectedCount(1); - runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); - - sleepUntilTimeChanges(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.READY, findChunkByIdOrThrow(chunkId).getStatus())); + myBatch2JobHelper.runMaintenancePass(); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, findChunkByIdOrThrow(chunkId).getStatus())); WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); mySvc.onWorkChunkFailed(chunkId, "This is an error message"); runInTransaction(() -> { - Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); + Batch2WorkChunkEntity entity = findChunkByIdOrThrow(chunkId); assertEquals(WorkChunkStatusEnum.FAILED, entity.getStatus()); assertEquals("This is an error message", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); @@ -612,6 +820,10 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); }); + latch.awaitExpected(); + verify(myBatchSender) + .sendWorkChannelMessage(any()); + clearInvocations(myBatchSender); } @Test @@ -626,7 +838,8 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { "stepId", instanceId, 0, - "{}" + "{}", + false ); String id = mySvc.onWorkChunkCreate(chunk); chunkIds.add(id); @@ -674,15 +887,57 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { .orElseThrow(IllegalArgumentException::new)); } + private JobInstance createInstance() { + return createInstance(false, false); + } @Nonnull - private JobInstance createInstance() { + private JobInstance createInstance(boolean theCreateJobDefBool, boolean theCreateGatedJob) { JobInstance instance = new JobInstance(); instance.setJobDefinitionId(JOB_DEFINITION_ID); instance.setStatus(StatusEnum.QUEUED); instance.setJobDefinitionVersion(JOB_DEF_VER); instance.setParameters(CHUNK_DATA); instance.setReport("TEST"); + + if (theCreateJobDefBool) { + JobDefinition jobDef; + + if (theCreateGatedJob) { + jobDef = TestJobDefinitionUtils.buildGatedJobDefinition( + JOB_DEFINITION_ID, + (step, sink) -> { + sink.accept(new FirstStepOutput()); + return RunOutcome.SUCCESS; + }, + (step, sink) -> { + return RunOutcome.SUCCESS; + }, + theDetails -> { + + } + ); + instance.setCurrentGatedStepId(jobDef.getFirstStepId()); + } else { + jobDef = TestJobDefinitionUtils.buildJobDefinition( + JOB_DEFINITION_ID, + (step, sink) -> { + sink.accept(new FirstStepOutput()); + return RunOutcome.SUCCESS; + }, + (step, sink) -> { + return RunOutcome.SUCCESS; + }, + theDetails -> { + + } + ); + } + if (myJobDefinitionRegistry.getJobDefinition(jobDef.getJobDefinitionId(), jobDef.getJobDefinitionVersion()).isEmpty()) { + myJobDefinitionRegistry.addJobDefinition(jobDef); + } + } + return instance; } @@ -719,4 +974,12 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { Arguments.of(WorkChunkStatusEnum.COMPLETED, false) ); } + + private Batch2JobInstanceEntity findInstanceByIdOrThrow(String instanceId) { + return myJobInstanceRepository.findById(instanceId).orElseThrow(IllegalStateException::new); + } + + private Batch2WorkChunkEntity findChunkByIdOrThrow(String secondChunkId) { + return myWorkChunkRepository.findById(secondChunkId).orElseThrow(IllegalArgumentException::new); + } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java index 1d0f89c493f..810c27bc900 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java @@ -13,7 +13,6 @@ import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import ca.uhn.fhir.jpa.batch2.JpaJobPersistenceImpl; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; -import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; import ca.uhn.fhir.rest.api.Constants; @@ -31,7 +30,6 @@ import ca.uhn.fhir.util.JsonUtil; import com.google.common.collect.Sets; import jakarta.annotation.Nonnull; import org.apache.commons.io.LineIterator; -import org.apache.commons.lang3.StringUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -72,7 +70,6 @@ import org.mockito.Spy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import java.io.IOException; import java.io.StringReader; @@ -85,10 +82,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; -import static ca.uhn.fhir.batch2.jobs.export.BulkExportAppCtx.CREATE_REPORT_STEP; -import static ca.uhn.fhir.batch2.jobs.export.BulkExportAppCtx.WRITE_TO_BINARIES; import static ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoR4TagsInlineTest.createSearchParameterForInlineSecurity; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.awaitility.Awaitility.await; @@ -477,7 +473,8 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { verifyBulkExportResults(options, ids, new ArrayList<>()); assertFalse(valueSet.isEmpty()); - assertEquals(ids.size(), valueSet.size()); + assertEquals(ids.size(), valueSet.size(), + "Expected " + String.join(", ", ids) + ". Actual : " + String.join(", ", valueSet)); for (String id : valueSet) { // should start with our value from the key-value pairs assertTrue(id.startsWith(value)); @@ -898,6 +895,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setResourceTypes(Sets.newHashSet("Patient", "Observation", "CarePlan", "MedicationAdministration", "ServiceRequest")); options.setExportStyle(BulkExportJobParameters.ExportStyle.PATIENT); options.setOutputFormat(Constants.CT_FHIR_NDJSON); + verifyBulkExportResults(options, List.of("Patient/P1", carePlanId, medAdminId, sevReqId, obsSubId, obsPerId), Collections.emptyList()); } @@ -1096,7 +1094,6 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { String resourceType = file.getKey(); List binaryIds = file.getValue(); for (var nextBinaryId : binaryIds) { - String nextBinaryIdPart = new IdType(nextBinaryId).getIdPart(); assertThat(nextBinaryIdPart, matchesPattern("[a-zA-Z0-9]{32}")); @@ -1105,6 +1102,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { String nextNdJsonFileContent = new String(binary.getContent(), Constants.CHARSET_UTF8); try (var iter = new LineIterator(new StringReader(nextNdJsonFileContent))) { + AtomicBoolean gate = new AtomicBoolean(false); iter.forEachRemaining(t -> { if (isNotBlank(t)) { IBaseResource next = myFhirContext.newJsonParser().parseResource(t); @@ -1117,7 +1115,10 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { } } } + gate.set(true); }); + await().atMost(400, TimeUnit.MILLISECONDS) + .until(gate::get); } catch (IOException e) { fail(e.toString()); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSystemDaoR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSystemDaoR4Test.java index 93dc9dfc6e3..a628a79539b 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSystemDaoR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSystemDaoR4Test.java @@ -93,7 +93,6 @@ import org.springframework.transaction.support.TransactionTemplate; import jakarta.annotation.Nonnull; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -582,13 +581,13 @@ public class FhirSystemDaoR4Test extends BaseJpaR4SystemTest { p.addName().setFamily("family"); final IIdType id = myPatientDao.create(p, mySrd).getId().toUnqualified(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); ValueSet vs = new ValueSet(); vs.setUrl("http://foo"); myValueSetDao.create(vs, mySrd); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); ResourceTable entity = new TransactionTemplate(myTxManager).execute(t -> myEntityManager.find(ResourceTable.class, id.getIdPartAsLong())); assertEquals(Long.valueOf(1), entity.getIndexStatus()); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java index 77cbd0d21ce..d033d703d17 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java @@ -18,7 +18,10 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.PatientReindexTestHelper; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; +import jakarta.annotation.PostConstruct; +import jakarta.persistence.Query; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Patient; @@ -30,8 +33,6 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; -import jakarta.annotation.PostConstruct; -import jakarta.persistence.Query; import java.util.Date; import java.util.List; import java.util.stream.Stream; @@ -263,7 +264,7 @@ public class ReindexJobTest extends BaseJpaR4Test { .setOptimizeStorage(ReindexParameters.OptimizeStorageModeEnum.CURRENT_VERSION) .setReindexSearchParameters(ReindexParameters.ReindexSearchParametersEnum.NONE) ); - Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(startRequest); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), startRequest); JobInstance outcome = myBatch2JobHelper.awaitJobCompletion(startResponse); assertEquals(10, outcome.getCombinedRecordsProcessed()); @@ -358,7 +359,7 @@ public class ReindexJobTest extends BaseJpaR4Test { myReindexTestHelper.createObservationWithAlleleExtension(Observation.ObservationStatus.FINAL); } - sleepUntilTimeChanges(); + sleepUntilTimeChange(); myReindexTestHelper.createAlleleSearchParameter(); mySearchParamRegistry.forceRefresh(); @@ -390,7 +391,7 @@ public class ReindexJobTest extends BaseJpaR4Test { JobInstanceStartRequest startRequest = new JobInstanceStartRequest(); startRequest.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX); startRequest.setParameters(new ReindexJobParameters()); - Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(startRequest); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), startRequest); JobInstance myJob = myBatch2JobHelper.awaitJobCompletion(startResponse); assertEquals(StatusEnum.COMPLETED, myJob.getStatus()); @@ -445,7 +446,7 @@ public class ReindexJobTest extends BaseJpaR4Test { JobInstanceStartRequest startRequest = new JobInstanceStartRequest(); startRequest.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX); startRequest.setParameters(new ReindexJobParameters()); - Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(startRequest); + Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(new SystemRequestDetails(), startRequest); JobInstance outcome = myBatch2JobHelper.awaitJobFailure(startResponse); // Verify diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java index ac93db3d643..422db3d6fe7 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java @@ -82,6 +82,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -440,8 +441,9 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes }.setValidationSupport(myValidationSupport)); // Should be ok - myClient.read().resource(Observation.class).withId("Observation/allowed").execute(); + Observation result = myClient.read().resource(Observation.class).withId("Observation/allowed").execute(); + assertNotNull(result); } @Test @@ -463,8 +465,10 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes }.setValidationSupport(myValidationSupport)); // Should be ok - myClient.read().resource(Patient.class).withId("Patient/P").execute(); - myClient.read().resource(Observation.class).withId("Observation/O").execute(); + Patient pat = myClient.read().resource(Patient.class).withId("Patient/P").execute(); + Observation obs = myClient.read().resource(Observation.class).withId("Observation/O").execute(); + assertNotNull(pat); + assertNotNull(obs); } /** diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderCustomSearchParamR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderCustomSearchParamR4Test.java index f6b9cd4b9df..b0239d24ee3 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderCustomSearchParamR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderCustomSearchParamR4Test.java @@ -244,12 +244,15 @@ public class ResourceProviderCustomSearchParamR4Test extends BaseResourceProvide mySearchParameterDao.create(fooSp, mySrd); runInTransaction(() -> { + myBatch2JobHelper.forceRunMaintenancePass(); + List allJobs = myBatch2JobHelper.findJobsByDefinition(ReindexAppCtx.JOB_REINDEX); assertEquals(1, allJobs.size()); assertEquals(1, allJobs.get(0).getParameters(ReindexJobParameters.class).getPartitionedUrls().size()); assertEquals("Patient?", allJobs.get(0).getParameters(ReindexJobParameters.class).getPartitionedUrls().get(0).getUrl()); }); + myBatch2JobHelper.awaitNoJobsRunning(); } @Test diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4BundleTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4BundleTest.java index 2b9f3891249..1136f192cc9 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4BundleTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4BundleTest.java @@ -3,9 +3,11 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; +import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.rest.server.exceptions.NotImplementedOperationException; import com.google.common.base.Charsets; import org.apache.commons.io.IOUtils; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; @@ -24,19 +26,32 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderR4BundleTest.class); + private static final int DESIRED_MAX_THREADS = 5; + + static { + if (TestR4Config.ourMaxThreads == null || TestR4Config.ourMaxThreads < DESIRED_MAX_THREADS) { + TestR4Config.ourMaxThreads = DESIRED_MAX_THREADS; + } + } + @BeforeEach @Override public void before() throws Exception { @@ -52,6 +67,7 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { myStorageSettings.setBundleBatchPoolSize(JpaStorageSettings.DEFAULT_BUNDLE_BATCH_POOL_SIZE); myStorageSettings.setBundleBatchMaxPoolSize(JpaStorageSettings.DEFAULT_BUNDLE_BATCH_MAX_POOL_SIZE); } + /** * See #401 */ @@ -69,14 +85,13 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { Bundle retBundle = myClient.read().resource(Bundle.class).withId(id).execute(); - ourLog.debug(myFhirContext.newXmlParser().setPrettyPrint(true).encodeResourceToString(retBundle)); + ourLog.debug(myFhirContext.newXmlParser().setPrettyPrint(true).encodeResourceToString(retBundle)); assertEquals("http://foo/", bundle.getEntry().get(0).getFullUrl()); } @Test public void testProcessMessage() { - Bundle bundle = new Bundle(); bundle.setType(BundleType.MESSAGE); @@ -117,22 +132,41 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { } - @Test - public void testHighConcurrencyWorks() throws IOException, InterruptedException { + public void testHighConcurrencyWorks() throws IOException { List bundles = new ArrayList<>(); for (int i =0 ; i < 10; i ++) { bundles.add(myFhirContext.newJsonParser().parseResource(Bundle.class, IOUtils.toString(getClass().getResourceAsStream("/r4/identical-tags-batch.json"), Charsets.UTF_8))); } - ExecutorService tpe = Executors.newFixedThreadPool(4); - for (Bundle bundle :bundles) { - tpe.execute(() -> myClient.transaction().withBundle(bundle).execute()); - } - tpe.shutdown(); - tpe.awaitTermination(100, TimeUnit.SECONDS); - } + int desiredMaxThreads = DESIRED_MAX_THREADS - 1; + int maxThreads = TestR4Config.getMaxThreads(); + // we want strictly > because we want at least 1 extra thread hanging around for + // any spun off processes needed internally during the transaction + assertTrue(maxThreads > desiredMaxThreads, String.format("Wanted > %d threads, but we only have %d available", desiredMaxThreads, maxThreads)); + ExecutorService tpe = Executors.newFixedThreadPool(desiredMaxThreads); + CompletionService completionService = new ExecutorCompletionService<>(tpe); + for (Bundle bundle : bundles) { + completionService.submit(() -> myClient.transaction().withBundle(bundle).execute()); + } + + int count = 0; + int expected = bundles.size(); + while (count < expected) { + try { + completionService.take(); + count++; + } catch (Exception ex) { + ourLog.error(ex.getMessage()); + fail(ex.getMessage()); + } + } + + tpe.shutdown(); + await().atMost(100, TimeUnit.SECONDS) + .until(tpe::isShutdown); + } @Test public void testBundleBatchWithSingleThread() { @@ -144,8 +178,9 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { Bundle input = new Bundle(); input.setType(BundleType.BATCH); - for (String id : ids) - input.addEntry().getRequest().setMethod(HTTPVerb.GET).setUrl(id); + for (String id : ids) { + input.addEntry().getRequest().setMethod(HTTPVerb.GET).setUrl(id); + } Bundle output = myClient.transaction().withBundle(input).execute(); @@ -158,9 +193,8 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { for (BundleEntryComponent bundleEntry : bundleEntries) { assertEquals(ids.get(i++), bundleEntry.getResource().getIdElement().toUnqualifiedVersionless().getValueAsString()); } - - } + @Test public void testBundleBatchWithError() { List ids = createPatients(5); @@ -351,7 +385,8 @@ public class ResourceProviderR4BundleTest extends BaseResourceProviderR4Test { bundle.getEntry().forEach(entry -> carePlans.add((CarePlan) entry.getResource())); // Post CarePlans should not get: HAPI-2006: Unable to perform PUT, URL provided is invalid... - myClient.transaction().withResources(carePlans).execute(); + List result = myClient.transaction().withResources(carePlans).execute(); + assertFalse(result.isEmpty()); } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CodeSystemTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CodeSystemTest.java index 06693388bee..c7dff314553 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CodeSystemTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CodeSystemTest.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.term.TermTestUtil; +import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import org.apache.commons.io.IOUtils; @@ -26,10 +27,13 @@ import org.hl7.fhir.r4.model.UriType; import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.io.IOException; +import java.util.concurrent.TimeUnit; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -37,12 +41,16 @@ import static org.junit.jupiter.api.Assertions.fail; public class ResourceProviderR4CodeSystemTest extends BaseResourceProviderR4Test { + private static final String SYSTEM_PARENTCHILD = "http://parentchild"; private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderR4CodeSystemTest.class); private static final String CS_ACME_URL = "http://acme.org"; private Long parentChildCsId; private IIdType myCsId; + @Autowired + private ITermDeferredStorageSvc myITermDeferredStorageSvc; + @BeforeEach @Transactional public void before02() throws IOException { @@ -63,6 +71,13 @@ public class ResourceProviderR4CodeSystemTest extends BaseResourceProviderR4Test DaoMethodOutcome parentChildCsOutcome = myCodeSystemDao.create(parentChildCs); parentChildCsId = ((ResourceTable) parentChildCsOutcome.getEntity()).getId(); + // ensure all terms are loaded + await().atMost(5, TimeUnit.SECONDS) + .until(() -> { + myBatch2JobHelper.forceRunMaintenancePass(); + myITermDeferredStorageSvc.saveDeferred(); + return myITermDeferredStorageSvc.isStorageQueueEmpty(true); + }); } @Test diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java index 2fcf5b19b1e..1cd96f9a46a 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java @@ -30,22 +30,22 @@ public class ResourceReindexSvcImplTest extends BaseJpaR4Test { // Setup createPatient(withActiveFalse()); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Date start = new Date(); Long id0 = createPatient(withActiveFalse()).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Long id1 = createPatient(withActiveFalse()).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Date beforeLastInRange = new Date(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Long id2 = createObservation(withObservationCode("http://foo", "bar")).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Date end = new Date(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); createPatient(withActiveFalse()); @@ -103,26 +103,26 @@ public class ResourceReindexSvcImplTest extends BaseJpaR4Test { // Setup final Long patientId0 = createPatient(withActiveFalse()).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); // Start of resources within range Date start = new Date(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Long patientId1 = createPatient(withActiveFalse()).getIdPartAsLong(); createObservation(withObservationCode("http://foo", "bar")); createObservation(withObservationCode("http://foo", "bar")); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Date beforeLastInRange = new Date(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Long patientId2 = createPatient(withActiveFalse()).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); Date end = new Date(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); // End of resources within range createObservation(withObservationCode("http://foo", "bar")); final Long patientId3 = createPatient(withActiveFalse()).getIdPartAsLong(); - sleepUntilTimeChanges(); + sleepUntilTimeChange(); // Execute diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobTest.java index 9b541282fa0..7f7897869d2 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobTest.java @@ -31,6 +31,7 @@ import ca.uhn.fhir.jpa.term.ZipCollectionBuilder; import ca.uhn.fhir.jpa.term.models.TermCodeSystemDeleteJobParameters; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.Batch2JobHelper; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.JsonUtil; @@ -127,7 +128,7 @@ public class TermCodeSystemDeleteJobTest extends BaseJpaR4Test { JobInstanceStartRequest request = new JobInstanceStartRequest(); request.setJobDefinitionId(TERM_CODE_SYSTEM_DELETE_JOB_NAME); request.setParameters(JsonUtil.serialize(parameters)); - Batch2JobStartResponse response = myJobCoordinator.startInstance(request); + Batch2JobStartResponse response = myJobCoordinator.startInstance(new SystemRequestDetails(), request); myBatch2JobHelper.awaitJobCompletion(response); @@ -147,7 +148,7 @@ public class TermCodeSystemDeleteJobTest extends BaseJpaR4Test { request.setParameters(new TermCodeSystemDeleteJobParameters()); // no pid InvalidRequestException exception = assertThrows(InvalidRequestException.class, () -> { - myJobCoordinator.startInstance(request); + myJobCoordinator.startInstance(new SystemRequestDetails(), request); }); assertTrue(exception.getMessage().contains("Invalid Term Code System PID 0"), exception.getMessage()); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/TestJobDefinitionUtils.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/TestJobDefinitionUtils.java new file mode 100644 index 00000000000..230edc6881c --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/TestJobDefinitionUtils.java @@ -0,0 +1,67 @@ +package ca.uhn.fhir.testjob; + +import ca.uhn.fhir.batch2.api.IJobCompletionHandler; +import ca.uhn.fhir.batch2.api.IJobStepWorker; +import ca.uhn.fhir.batch2.api.VoidModel; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.testjob.models.FirstStepOutput; +import ca.uhn.fhir.testjob.models.TestJobParameters; + +@SuppressWarnings({"unchecked", "rawtypes"}) +public class TestJobDefinitionUtils { + + public static final int TEST_JOB_VERSION = 1; + public static final String FIRST_STEP_ID = "first-step"; + public static final String LAST_STEP_ID = "last-step"; + + /** + * Creates a test job definition. + * This job will not be gated. + */ + public static JobDefinition buildJobDefinition( + String theJobId, + IJobStepWorker theFirstStep, + IJobStepWorker theLastStep, + IJobCompletionHandler theCompletionHandler) { + return getJobBuilder(theJobId, theFirstStep, theLastStep, theCompletionHandler).build(); + } + + /** + * Creates a test job defintion. + * This job will be gated. + */ + public static JobDefinition buildGatedJobDefinition( + String theJobId, + IJobStepWorker theFirstStep, + IJobStepWorker theLastStep, + IJobCompletionHandler theCompletionHandler) { + return getJobBuilder(theJobId, theFirstStep, theLastStep, theCompletionHandler) + .gatedExecution().build(); + } + + private static JobDefinition.Builder getJobBuilder( + String theJobId, + IJobStepWorker theFirstStep, + IJobStepWorker theLastStep, + IJobCompletionHandler theCompletionHandler + ) { + return JobDefinition.newBuilder() + .setJobDefinitionId(theJobId) + .setJobDescription("test job") + .setJobDefinitionVersion(TEST_JOB_VERSION) + .setParametersType(TestJobParameters.class) + .addFirstStep( + FIRST_STEP_ID, + "Test first step", + FirstStepOutput.class, + theFirstStep + ) + .addLastStep( + LAST_STEP_ID, + "Test last step", + theLastStep + ) + .completionHandler(theCompletionHandler); + } +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/FirstStepOutput.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/FirstStepOutput.java new file mode 100644 index 00000000000..34cefc682f8 --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/FirstStepOutput.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.testjob.models; + +import ca.uhn.fhir.model.api.IModelJson; + +/** + * Sample first step output for test job defintions created in {@link ca.uhn.fhir.testjob.TestJobDefinitionUtils} + */ +public class FirstStepOutput implements IModelJson { +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/ReductionStepOutput.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/ReductionStepOutput.java new file mode 100644 index 00000000000..62e2a101188 --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/ReductionStepOutput.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.testjob.models; + +import ca.uhn.fhir.model.api.IModelJson; + +/** + * Sample output object for reduction steps for test job created in {@link ca.uhn.fhir.testjob.TestJobDefinitionUtils} + */ +public class ReductionStepOutput implements IModelJson { +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/TestJobParameters.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/TestJobParameters.java new file mode 100644 index 00000000000..6fb3aa8650c --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/testjob/models/TestJobParameters.java @@ -0,0 +1,9 @@ +package ca.uhn.fhir.testjob.models; + +import ca.uhn.fhir.model.api.IModelJson; + +/** + * Sample job parameters; these are used for jobs created in {@link ca.uhn.fhir.testjob.TestJobDefinitionUtils} + */ +public class TestJobParameters implements IModelJson { +} diff --git a/hapi-fhir-jpaserver-test-r4b/pom.xml b/hapi-fhir-jpaserver-test-r4b/pom.xml index 8a2cd86b95d..0c26dd83b61 100644 --- a/hapi-fhir-jpaserver-test-r4b/pom.xml +++ b/hapi-fhir-jpaserver-test-r4b/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r5/pom.xml b/hapi-fhir-jpaserver-test-r5/pom.xml index 0c7622cb1df..b3dc3455ff3 100644 --- a/hapi-fhir-jpaserver-test-r5/pom.xml +++ b/hapi-fhir-jpaserver-test-r5/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/pom.xml b/hapi-fhir-jpaserver-test-utilities/pom.xml index b756a6dbbd0..5aad3443ecb 100644 --- a/hapi-fhir-jpaserver-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java index 33d536600c1..ae49003702d 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java @@ -19,6 +19,7 @@ */ package ca.uhn.fhir.jpa.test; +import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.jobs.export.BulkDataExportProvider; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; @@ -218,6 +219,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.fail; @ExtendWith(SpringExtension.class) @@ -247,7 +249,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @Autowired protected ISearchDao mySearchEntityDao; @Autowired - private IBatch2JobInstanceRepository myJobInstanceRepository; + protected IBatch2JobInstanceRepository myJobInstanceRepository; @Autowired private IBatch2WorkChunkRepository myWorkChunkRepository; @@ -553,11 +555,18 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @Autowired protected TestDaoSearch myTestDaoSearch; + @Autowired + protected IJobMaintenanceService myJobMaintenanceService; + @RegisterExtension private final PreventDanglingInterceptorsExtension myPreventDanglingInterceptorsExtension = new PreventDanglingInterceptorsExtension(()-> myInterceptorRegistry); @AfterEach() + @Order(0) public void afterCleanupDao() { + // make sure there are no running jobs + assertFalse(myBatch2JobHelper.hasRunningJobs()); + myStorageSettings.setExpireSearchResults(new JpaStorageSettings().isExpireSearchResults()); myStorageSettings.setEnforceReferentialIntegrityOnDelete(new JpaStorageSettings().isEnforceReferentialIntegrityOnDelete()); myStorageSettings.setExpireSearchResultsAfterMillis(new JpaStorageSettings().getExpireSearchResultsAfterMillis()); @@ -572,6 +581,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil myPagingProvider.setMaximumPageSize(BasePagingProvider.DEFAULT_MAX_PAGE_SIZE); myPartitionSettings.setPartitioningEnabled(false); + ourLog.info("1 - " + getClass().getSimpleName() + ".afterCleanupDao"); } @Override @@ -580,6 +590,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil public void afterResetInterceptors() { super.afterResetInterceptors(); myInterceptorRegistry.unregisterInterceptor(myPerformanceTracingLoggingInterceptor); + + ourLog.info("2 - " + getClass().getSimpleName() + ".afterResetInterceptors"); } @AfterEach @@ -590,6 +602,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil TermConceptMappingSvcImpl.clearOurLastResultsFromTranslationWithReverseCache(); TermDeferredStorageSvcImpl termDeferredStorageSvc = AopTestUtils.getTargetObject(myTerminologyDeferredStorageSvc); termDeferredStorageSvc.clearDeferred(); + + ourLog.info("4 - " + getClass().getSimpleName() + ".afterClearTerminologyCaches"); } @BeforeEach @@ -613,6 +627,21 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @AfterEach public void afterPurgeDatabase() { + /* + * We have to stop all scheduled jobs or they will + * interfere with the database cleanup! + */ + ourLog.info("Pausing Schedulers"); + mySchedulerService.pause(); + + myTerminologyDeferredStorageSvc.logQueueForUnitTest(); + if (!myTermDeferredStorageSvc.isStorageQueueEmpty(true)) { + ourLog.warn("There is deferred terminology storage stuff still in the queue. Please verify your tests clean up ok."); + if (myTermDeferredStorageSvc instanceof TermDeferredStorageSvcImpl t) { + t.clearDeferred(); + } + } + boolean registeredStorageInterceptor = false; if (myMdmStorageInterceptor != null && !myInterceptorService.getAllRegisteredInterceptors().contains(myMdmStorageInterceptor)) { myInterceptorService.registerInterceptor(myMdmStorageInterceptor); @@ -635,6 +664,11 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil myInterceptorService.unregisterInterceptor(myMdmStorageInterceptor); } } + + // restart the jobs + ourLog.info("Restarting the schedulers"); + mySchedulerService.unpause(); + ourLog.info("5 - " + getClass().getSimpleName() + ".afterPurgeDatabases"); } @BeforeEach @@ -819,6 +853,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @AfterEach public void afterEachClearCaches() { myJpaValidationSupportChainR4.invalidateCaches(); + ourLog.info("3 - " + getClass().getSimpleName() + ".afterEachClearCaches"); } private static void flattenExpansionHierarchy(List theFlattenedHierarchy, List theCodes, String thePrefix) { diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java index 8a6890d96ae..b8af987a9f2 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java @@ -69,6 +69,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri; import ca.uhn.fhir.jpa.model.entity.ResourceLink; import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.model.sched.ISchedulerService; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; @@ -77,6 +78,7 @@ import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.subscription.match.registry.SubscriptionLoader; import ca.uhn.fhir.jpa.subscription.match.registry.SubscriptionRegistry; +import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -243,6 +245,8 @@ public abstract class BaseJpaTest extends BaseTest { protected ITermConceptPropertyDao myTermConceptPropertyDao; @Autowired private MemoryCacheService myMemoryCacheService; + @Autowired + protected ISchedulerService mySchedulerService; @Qualifier(JpaConfig.JPA_VALIDATION_SUPPORT) @Autowired private IValidationSupport myJpaPersistedValidationSupport; @@ -256,6 +260,8 @@ public abstract class BaseJpaTest extends BaseTest { private IResourceHistoryTableDao myResourceHistoryTableDao; @Autowired private DaoRegistry myDaoRegistry; + @Autowired + protected ITermDeferredStorageSvc myTermDeferredStorageSvc; private final List myRegisteredInterceptors = new ArrayList<>(1); @SuppressWarnings("BusyWait") @@ -291,7 +297,7 @@ public abstract class BaseJpaTest extends BaseTest { } @SuppressWarnings("BusyWait") - protected static void purgeDatabase(JpaStorageSettings theStorageSettings, IFhirSystemDao theSystemDao, IResourceReindexingSvc theResourceReindexingSvc, ISearchCoordinatorSvc theSearchCoordinatorSvc, ISearchParamRegistry theSearchParamRegistry, IBulkDataExportJobSchedulingHelper theBulkDataJobActivator) { + public static void purgeDatabase(JpaStorageSettings theStorageSettings, IFhirSystemDao theSystemDao, IResourceReindexingSvc theResourceReindexingSvc, ISearchCoordinatorSvc theSearchCoordinatorSvc, ISearchParamRegistry theSearchParamRegistry, IBulkDataExportJobSchedulingHelper theBulkDataJobActivator) { theSearchCoordinatorSvc.cancelAllActiveSearches(); theResourceReindexingSvc.cancelAndPurgeAllJobs(); theBulkDataJobActivator.cancelAndPurgeAllJobs(); @@ -303,6 +309,7 @@ public abstract class BaseJpaTest extends BaseTest { for (int count = 0; ; count++) { try { + ourLog.info("Calling Expunge count {}", count); theSystemDao.expunge(new ExpungeOptions().setExpungeEverything(true), new SystemRequestDetails()); break; } catch (Exception e) { @@ -595,9 +602,9 @@ public abstract class BaseJpaTest extends BaseTest { } /** - * Sleep until at least 1 ms has elapsed + * Sleep until time change on the clocks */ - public void sleepUntilTimeChanges() { + public void sleepUntilTimeChange() { StopWatch sw = new StopWatch(); await().until(() -> sw.getMillis() > 0); } diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java index 204da1cd32e..37d58f02776 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java @@ -24,6 +24,7 @@ import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import org.awaitility.Awaitility; import org.awaitility.core.ConditionTimeoutException; @@ -32,10 +33,13 @@ import org.slf4j.LoggerFactory; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.thymeleaf.util.ArrayUtils; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.awaitility.Awaitility.await; @@ -89,10 +93,12 @@ public class Batch2JobHelper { public JobInstance awaitJobHasStatus(String theInstanceId, int theSecondsToWait, StatusEnum... theExpectedStatus) { assert !TransactionSynchronizationManager.isActualTransactionActive(); + AtomicInteger checkCount = new AtomicInteger(); try { await() .atMost(theSecondsToWait, TimeUnit.SECONDS) .until(() -> { + checkCount.getAndIncrement(); boolean inFinalStatus = false; if (ArrayUtils.contains(theExpectedStatus, StatusEnum.COMPLETED) && !ArrayUtils.contains(theExpectedStatus, StatusEnum.FAILED)) { inFinalStatus = hasStatus(theInstanceId, StatusEnum.FAILED); @@ -113,7 +119,9 @@ public class Batch2JobHelper { .map(t -> t.getInstanceId() + " " + t.getJobDefinitionId() + "/" + t.getStatus().name()) .collect(Collectors.joining("\n")); String currentStatus = myJobCoordinator.getInstance(theInstanceId).getStatus().name(); - fail("Job " + theInstanceId + " still has status " + currentStatus + " - All statuses:\n" + statuses); + fail("Job " + theInstanceId + " still has status " + currentStatus + + " after " + checkCount.get() + " checks in " + theSecondsToWait + " seconds." + + " - All statuses:\n" + statuses); } return myJobCoordinator.getInstance(theInstanceId); } @@ -162,8 +170,39 @@ public class Batch2JobHelper { return awaitJobHasStatus(theInstanceId, StatusEnum.ERRORED, StatusEnum.FAILED); } + public void awaitJobHasStatusWithForcedMaintenanceRuns(String theInstanceId, StatusEnum theStatusEnum) { + AtomicInteger counter = new AtomicInteger(); + try { + await() + .atMost(Duration.of(10, ChronoUnit.SECONDS)) + .until(() -> { + counter.getAndIncrement(); + forceRunMaintenancePass(); + return hasStatus(theInstanceId, theStatusEnum); + }); + } catch (ConditionTimeoutException ex) { + StatusEnum status = getStatus(theInstanceId); + String msg = String.format( + "Job %s has state %s after 10s timeout and %d checks", + theInstanceId, + status.name(), + counter.get() + ); + } + } + public void awaitJobInProgress(String theInstanceId) { - await().until(() -> checkStatusWithMaintenancePass(theInstanceId, StatusEnum.IN_PROGRESS)); + try { + await() + .atMost(Duration.of(10, ChronoUnit.SECONDS)) + .until(() -> checkStatusWithMaintenancePass(theInstanceId, StatusEnum.IN_PROGRESS)); + } catch (ConditionTimeoutException ex) { + StatusEnum statusEnum = getStatus(theInstanceId); + String msg = String.format("Job %s still has status %s after 10 seconds.", + theInstanceId, + statusEnum.name()); + fail(msg); + } } public void assertNotFastTracking(String theInstanceId) { @@ -175,7 +214,21 @@ public class Batch2JobHelper { } public void awaitGatedStepId(String theExpectedGatedStepId, String theInstanceId) { - await().until(() -> theExpectedGatedStepId.equals(myJobCoordinator.getInstance(theInstanceId).getCurrentGatedStepId())); + try { + await().until(() -> { + String currentGatedStepId = myJobCoordinator.getInstance(theInstanceId).getCurrentGatedStepId(); + return theExpectedGatedStepId.equals(currentGatedStepId); + }); + } catch (ConditionTimeoutException ex) { + JobInstance instance = myJobCoordinator.getInstance(theInstanceId); + String msg = String.format("Instance %s of Job %s never got to step %s. Current step %s, current status %s.", + theInstanceId, + instance.getJobDefinitionId(), + theExpectedGatedStepId, + instance.getCurrentGatedStepId(), + instance.getStatus().name()); + fail(msg); + } } public long getCombinedRecordsProcessed(String theInstanceId) { @@ -223,6 +276,33 @@ public class Batch2JobHelper { awaitNoJobsRunning(false); } + public boolean hasRunningJobs() { + HashMap map = new HashMap<>(); + List jobs = myJobCoordinator.getInstances(1000, 1); + // "All Jobs" assumes at least one job exists + if (jobs.isEmpty()) { + return false; + } + + for (JobInstance job : jobs) { + if (job.getStatus().isIncomplete()) { + map.put(job.getInstanceId(), job.getJobDefinitionId() + " : " + job.getStatus().name()); + } + } + + if (!map.isEmpty()) { + ourLog.error( + "Found Running Jobs " + + map.keySet().stream() + .map(k -> k + " : " + map.get(k)) + .collect(Collectors.joining("\n")) + ); + + return true; + } + return false; + } + public void awaitNoJobsRunning(boolean theExpectAtLeastOneJobToExist) { HashMap map = new HashMap<>(); Awaitility.await().atMost(10, TimeUnit.SECONDS) @@ -255,6 +335,10 @@ public class Batch2JobHelper { myJobMaintenanceService.runMaintenancePass(); } + public void enableMaintenanceRunner(boolean theEnabled) { + myJobMaintenanceService.enableMaintenancePass(theEnabled); + } + /** * Forces a run of the maintenance pass without waiting for * the semaphore to release diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/Batch2FastSchedulerConfig.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/Batch2FastSchedulerConfig.java new file mode 100644 index 00000000000..28e3ae6ea1d --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/Batch2FastSchedulerConfig.java @@ -0,0 +1,23 @@ +package ca.uhn.fhir.jpa.test.config; + +import ca.uhn.fhir.batch2.api.IJobMaintenanceService; +import ca.uhn.fhir.batch2.maintenance.JobMaintenanceServiceImpl; +import jakarta.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; + +/** + * A fast scheduler to use for Batch2 job Integration Tests. + * This scheduler will run every 200ms (instead of the default 1min) + * so that our ITs can complete in a sane amount of time. + */ +@Configuration +public class Batch2FastSchedulerConfig { + @Autowired + IJobMaintenanceService myJobMaintenanceService; + + @PostConstruct + void fastScheduler() { + ((JobMaintenanceServiceImpl)myJobMaintenanceService).setScheduledJobFrequencyMillis(200); + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java index 3bae2e3ad1f..7fd84089838 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java @@ -57,11 +57,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -85,6 +81,7 @@ import static org.junit.jupiter.api.Assertions.fail; public class TestR4Config { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TestR4Config.class); + public static Integer ourMaxThreads; private final AtomicInteger myBorrowedConnectionCount = new AtomicInteger(0); private final AtomicInteger myReturnedConnectionCount = new AtomicInteger(0); @@ -96,7 +93,7 @@ public class TestR4Config { * starvation */ if (ourMaxThreads == null) { - ourMaxThreads = (int) (Math.random() * 6.0) + 3; + ourMaxThreads = (int) (Math.random() * 6.0) + 4; if (HapiTestSystemProperties.isSingleDbConnectionEnabled()) { ourMaxThreads = 1; @@ -108,7 +105,7 @@ public class TestR4Config { ourLog.warn("ourMaxThreads={}", ourMaxThreads); } - private Map myConnectionRequestStackTraces = Collections.synchronizedMap(new LinkedHashMap<>()); + private final Map myConnectionRequestStackTraces = Collections.synchronizedMap(new LinkedHashMap<>()); @Autowired TestHSearchAddInConfig.IHSearchConfigurer hibernateSearchConfigurer; @@ -300,5 +297,4 @@ public class TestR4Config { public static int getMaxThreads() { return ourMaxThreads; } - } diff --git a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml index cd7ce2cb444..bd83c065399 100644 --- a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml +++ b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-server-cds-hooks/pom.xml b/hapi-fhir-server-cds-hooks/pom.xml index 10ad59331cd..d5de58458b9 100644 --- a/hapi-fhir-server-cds-hooks/pom.xml +++ b/hapi-fhir-server-cds-hooks/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server-mdm/pom.xml b/hapi-fhir-server-mdm/pom.xml index a012bafd9b5..8be4eeef20e 100644 --- a/hapi-fhir-server-mdm/pom.xml +++ b/hapi-fhir-server-mdm/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server-openapi/pom.xml b/hapi-fhir-server-openapi/pom.xml index a152e68cf96..91b0ba08703 100644 --- a/hapi-fhir-server-openapi/pom.xml +++ b/hapi-fhir-server-openapi/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/pom.xml b/hapi-fhir-server/pom.xml index e81d4d6c257..ffce06e1c4a 100644 --- a/hapi-fhir-server/pom.xml +++ b/hapi-fhir-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml index b545d5e6ca6..60e63aab6ac 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml index a3eea68ca7a..8b0d686bf00 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml @@ -21,7 +21,7 @@ ca.uhn.hapi.fhir hapi-fhir-caching-api - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml index f009f0fb9e9..f932afeef76 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml index 0c262056280..e589b46af12 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml @@ -7,7 +7,7 @@ hapi-fhir ca.uhn.hapi.fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../pom.xml diff --git a/hapi-fhir-serviceloaders/pom.xml b/hapi-fhir-serviceloaders/pom.xml index fd2e15e3e75..fb776ba6387 100644 --- a/hapi-fhir-serviceloaders/pom.xml +++ b/hapi-fhir-serviceloaders/pom.xml @@ -5,7 +5,7 @@ hapi-deployable-pom ca.uhn.hapi.fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml index 7ac274198b7..c78e261e01a 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml index 07748abe01a..88f887381f2 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT hapi-fhir-spring-boot-sample-client-apache diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml index c3cdf23bf3c..206beef0160 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml index 3290e867a12..14cf689c7c2 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml index bd198dfe7ae..e37aaba366e 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml index 924eb57db31..af6174b4f59 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/pom.xml b/hapi-fhir-spring-boot/pom.xml index 2b661d2f417..4b909f3288e 100644 --- a/hapi-fhir-spring-boot/pom.xml +++ b/hapi-fhir-spring-boot/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-sql-migrate/pom.xml b/hapi-fhir-sql-migrate/pom.xml index 508e17fe930..88c51675f81 100644 --- a/hapi-fhir-sql-migrate/pom.xml +++ b/hapi-fhir-sql-migrate/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2-jobs/pom.xml b/hapi-fhir-storage-batch2-jobs/pom.xml index 71238102505..863a09e4eb6 100644 --- a/hapi-fhir-storage-batch2-jobs/pom.xml +++ b/hapi-fhir-storage-batch2-jobs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2-test-utilities/pom.xml b/hapi-fhir-storage-batch2-test-utilities/pom.xml index 8d57d1b7948..4f97d4da2b4 100644 --- a/hapi-fhir-storage-batch2-test-utilities/pom.xml +++ b/hapi-fhir-storage-batch2-test-utilities/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml @@ -17,6 +17,12 @@ HAPI FHIR JPA Server - Batch2 specification tests Batch2 is a framework for managing and executing long running "batch" jobs + + 17 + 17 + 17 + + ca.uhn.hapi.fhir diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java index 230944b128a..da7b528c8c4 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java @@ -20,24 +20,32 @@ package ca.uhn.hapi.fhir.batch2.test; +import ca.uhn.fhir.batch2.api.ChunkExecutionDetails; +import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.api.IJobPersistence; +import ca.uhn.fhir.batch2.api.IReductionStepWorker; +import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; +import ca.uhn.fhir.batch2.api.StepExecutionDetails; +import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.channel.BatchJobSender; import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; +import ca.uhn.fhir.batch2.model.ChunkOutcome; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; -import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.util.StopWatch; +import ca.uhn.hapi.fhir.batch2.test.support.JobMaintenanceStateInformation; import ca.uhn.hapi.fhir.batch2.test.support.TestJobParameters; import ca.uhn.hapi.fhir.batch2.test.support.TestJobStep2InputType; import ca.uhn.hapi.fhir.batch2.test.support.TestJobStep3InputType; import ca.uhn.test.concurrency.PointcutLatch; import jakarta.annotation.Nonnull; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Nested; import org.mockito.ArgumentCaptor; @@ -64,7 +72,8 @@ import static org.mockito.Mockito.verify; * These tests are abstract, and do not depend on JPA. * Test setups should use the public batch2 api to create scenarios. */ -public abstract class AbstractIJobPersistenceSpecificationTest implements IInProgressActionsTests, IInstanceStateTransitions, ITestFixture, WorkChunkTestConstants { +public abstract class AbstractIJobPersistenceSpecificationTest + implements ITestFixture, IWorkChunkCommon, WorkChunkTestConstants, IJobMaintenanceActions, IInProgressActionsTests, IInstanceStateTransitions { private static final Logger ourLog = LoggerFactory.getLogger(AbstractIJobPersistenceSpecificationTest.class); @@ -91,38 +100,63 @@ public abstract class AbstractIJobPersistenceSpecificationTest implements IInPro return mySvc; } - public JobDefinition withJobDefinition(boolean theIsGatedBoolean) { + @Nonnull + public JobDefinition withJobDefinitionWithReductionStep() { JobDefinition.Builder builder = JobDefinition.newBuilder() - .setJobDefinitionId(theIsGatedBoolean ? GATED_JOB_DEFINITION_ID : JOB_DEFINITION_ID) + .setJobDefinitionId(GATED_JOB_DEFINITION_ID + "_reduction") .setJobDefinitionVersion(JOB_DEF_VER) + .gatedExecution() .setJobDescription("A job description") .setParametersType(TestJobParameters.class) - .addFirstStep(TARGET_STEP_ID, "the first step", TestJobStep2InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) - .addIntermediateStep("2nd-step-id", "the second step", TestJobStep3InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) - .addLastStep("last-step-id", "the final step", (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)); + .addFirstStep(FIRST_STEP_ID, "the first step", TestJobStep2InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addIntermediateStep(SECOND_STEP_ID, "the second step", TestJobStep3InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addFinalReducerStep(LAST_STEP_ID, "reduction step", VoidModel.class, new IReductionStepWorker() { + @NotNull + @Override + public ChunkOutcome consume(ChunkExecutionDetails theChunkDetails) { + return ChunkOutcome.SUCCESS(); + } + + @NotNull + @Override + public RunOutcome run(@NotNull StepExecutionDetails theStepExecutionDetails, @NotNull IJobDataSink theDataSink) throws JobExecutionFailedException { + return RunOutcome.SUCCESS; + } + }); + return builder.build(); + } + + @Nonnull + public JobDefinition withJobDefinition(boolean theIsGatedBoolean) { + JobDefinition.Builder builder = JobDefinition.newBuilder() + .setJobDefinitionId(theIsGatedBoolean ? GATED_JOB_DEFINITION_ID : JOB_DEFINITION_ID) + .setJobDefinitionVersion(JOB_DEF_VER) + .setJobDescription("A job description") + .setParametersType(TestJobParameters.class) + .addFirstStep(FIRST_STEP_ID, "the first step", TestJobStep2InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addIntermediateStep(SECOND_STEP_ID, "the second step", TestJobStep3InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addLastStep(LAST_STEP_ID, "the final step", (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)); if (theIsGatedBoolean) { builder.gatedExecution(); } return builder.build(); } - @AfterEach - public void after() { - myJobDefinitionRegistry.removeJobDefinition(JOB_DEFINITION_ID, JOB_DEF_VER); - - // clear invocations on the batch sender from previous jobs that might be - // kicking around - Mockito.clearInvocations(myBatchJobSender); - } - @Override public ITestFixture getTestManager() { return this; } - @Override - public void enableMaintenanceRunner(boolean theToEnable) { - myMaintenanceService.enableMaintenancePass(theToEnable); + @AfterEach + public void after() { + myJobDefinitionRegistry.removeJobDefinition(JOB_DEFINITION_ID, JOB_DEF_VER); + + // re-enable our runner after every test (just in case) + myMaintenanceService.enableMaintenancePass(true); + + // clear invocations on the batch sender from previous jobs that might be + // kicking around + Mockito.clearInvocations(myBatchJobSender); } @Nested @@ -167,11 +201,19 @@ public abstract class AbstractIJobPersistenceSpecificationTest implements IInPro instance.setJobDefinitionVersion(JOB_DEF_VER); instance.setParameters(CHUNK_DATA); instance.setReport("TEST"); + if (jobDefinition.isGatedExecution()) { + instance.setCurrentGatedStepId(jobDefinition.getFirstStepId()); + } return instance; } - public String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData) { - WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData); + public String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData, boolean theGatedExecution) { + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData, theGatedExecution); + return mySvc.onWorkChunkCreate(batchWorkChunk); + } + + public String storeFirstWorkChunk(JobDefinition theJobDefinition, String theInstanceId) { + WorkChunkCreateEvent batchWorkChunk = WorkChunkCreateEvent.firstChunk(theJobDefinition, theInstanceId); return mySvc.onWorkChunkCreate(batchWorkChunk); } @@ -230,11 +272,20 @@ public abstract class AbstractIJobPersistenceSpecificationTest implements IInPro return chunkId; } - @Override - public abstract WorkChunk freshFetchWorkChunk(String theChunkId); - public String createChunk(String theInstanceId) { - return storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, theInstanceId, 0, CHUNK_DATA); + return storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, theInstanceId, 0, CHUNK_DATA, false); + } + + public String createChunk(String theInstanceId, boolean theGatedExecution) { + return storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, theInstanceId, 0, CHUNK_DATA, theGatedExecution); + } + + public String createFirstChunk(JobDefinition theJobDefinition, String theJobInstanceId){ + return storeFirstWorkChunk(theJobDefinition, theJobInstanceId); + } + + public void enableMaintenanceRunner(boolean theToEnable) { + myMaintenanceService.enableMaintenancePass(theToEnable); } public PointcutLatch disableWorkChunkMessageHandler() { @@ -254,4 +305,9 @@ public abstract class AbstractIJobPersistenceSpecificationTest implements IInPro verify(myBatchJobSender, times(theNumberOfTimes)) .sendWorkChannelMessage(notificationCaptor.capture()); } + + @Override + public void createChunksInStates(JobMaintenanceStateInformation theJobMaintenanceStateInformation) { + theJobMaintenanceStateInformation.initialize(mySvc); + } } diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IInstanceStateTransitions.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IInstanceStateTransitions.java index 377b603eabb..89e99aaa125 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IInstanceStateTransitions.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IInstanceStateTransitions.java @@ -26,6 +26,8 @@ import ca.uhn.fhir.batch2.maintenance.JobInstanceProcessor; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -34,6 +36,9 @@ import org.slf4j.LoggerFactory; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -42,6 +47,31 @@ import static org.junit.jupiter.api.Assertions.assertNull; public interface IInstanceStateTransitions extends IWorkChunkCommon, WorkChunkTestConstants { Logger ourLog = LoggerFactory.getLogger(IInstanceStateTransitions.class); + @Test + default void createInstance_createsInQueuedWithChunkInReady() { + // given + JobDefinition jd = getTestManager().withJobDefinition(false); + + // when + IJobPersistence.CreateResult createResult = + getTestManager().newTxTemplate().execute(status-> + getTestManager().getSvc().onCreateWithFirstChunk(jd, "{}")); + + // then + ourLog.info("job and chunk created {}", createResult); + assertNotNull(createResult); + assertThat(createResult.jobInstanceId, not(emptyString())); + assertThat(createResult.workChunkId, not(emptyString())); + + JobInstance jobInstance = getTestManager().freshFetchJobInstance(createResult.jobInstanceId); + assertThat(jobInstance.getStatus(), equalTo(StatusEnum.QUEUED)); + assertThat(jobInstance.getParameters(), equalTo("{}")); + + WorkChunk firstChunk = getTestManager().freshFetchWorkChunk(createResult.workChunkId); + assertThat(firstChunk.getStatus(), equalTo(WorkChunkStatusEnum.READY)); + assertNull(firstChunk.getData(), "First chunk data is null - only uses parameters"); + } + @Test default void testCreateInstance_firstChunkDequeued_movesToInProgress() { // given diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IJobMaintenanceActions.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IJobMaintenanceActions.java new file mode 100644 index 00000000000..cb561ad87a1 --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IJobMaintenanceActions.java @@ -0,0 +1,234 @@ +package ca.uhn.hapi.fhir.batch2.test; + +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.hapi.fhir.batch2.test.support.JobMaintenanceStateInformation; +import ca.uhn.test.concurrency.PointcutLatch; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public interface IJobMaintenanceActions extends IWorkChunkCommon, WorkChunkTestConstants { + + Logger ourLog = LoggerFactory.getLogger(IJobMaintenanceActions.class); + + @BeforeEach + default void before() { + getTestManager().enableMaintenanceRunner(false); + } + + @Test + default void test_gatedJob_stepReady_stepAdvances() throws InterruptedException { + // setup + String initialState = """ + # chunks ready - move to queued + 1|COMPLETED + 2|READY,2|QUEUED + 2|READY,2|QUEUED + """; + int numToTransition = 2; + PointcutLatch sendLatch = getTestManager().disableWorkChunkMessageHandler(); + sendLatch.setExpectedCount(numToTransition); + JobMaintenanceStateInformation result = setupGatedWorkChunkTransitionTest(initialState, true); + getTestManager().createChunksInStates(result); + + // test + getTestManager().runMaintenancePass(); + + // verify + getTestManager().verifyWorkChunkMessageHandlerCalled(sendLatch, numToTransition); + verifyWorkChunkFinalStates(result); + } + + @ParameterizedTest + @ValueSource(strings = { + """ + 1|COMPLETED + 2|GATE_WAITING + """, + """ + # Chunk already queued -> waiting for complete + 1|COMPLETED + 2|QUEUED + """, + """ + # Chunks in progress, complete, errored -> cannot advance + 1|COMPLETED + 2|COMPLETED + 2|ERRORED + 2|IN_PROGRESS + """, + """ + # Chunk in errored/already queued -> cannot advance + 1|COMPLETED + 2|ERRORED # equivalent of QUEUED + 2|COMPLETED + """, + """ + # Not all steps ready to advance + # Latch Count: 1 + 1|COMPLETED + 2|READY,2|QUEUED # a single ready chunk + 2|IN_PROGRESS + """, + """ + # Previous step not ready -> do not advance + 1|COMPLETED + 2|COMPLETED + 2|IN_PROGRESS + 3|GATE_WAITING + 3|GATE_WAITING + """, + """ + # when current step is not all queued, should queue READY chunks + # Latch Count: 1 + 1|COMPLETED + 2|READY,2|QUEUED + 2|QUEUED + 2|COMPLETED + 2|ERRORED + 2|FAILED + 2|IN_PROGRESS + 3|GATE_WAITING + 3|QUEUED + """, + """ + # when current step is all queued but not done, should not proceed + 1|COMPLETED + 2|COMPLETED + 2|QUEUED + 2|COMPLETED + 2|ERRORED + 2|FAILED + 2|IN_PROGRESS + 3|GATE_WAITING + 3|GATE_WAITING + """ + }) + default void testGatedStep2NotReady_stepNotAdvanceToStep3(String theChunkState) throws InterruptedException { + // setup + int expectedLatchCount = getLatchCountFromState(theChunkState); + PointcutLatch sendingLatch = getTestManager().disableWorkChunkMessageHandler(); + sendingLatch.setExpectedCount(expectedLatchCount); + JobMaintenanceStateInformation state = setupGatedWorkChunkTransitionTest(theChunkState, true); + + getTestManager().createChunksInStates(state); + + // test + getTestManager().runMaintenancePass(); + + // verify + // nothing ever queued -> nothing ever sent to queue + getTestManager().verifyWorkChunkMessageHandlerCalled(sendingLatch, expectedLatchCount); + assertEquals(SECOND_STEP_ID, getJobInstanceFromState(state).getCurrentGatedStepId()); + verifyWorkChunkFinalStates(state); + } + + /** + * Returns the expected latch count specified in the state. Defaults to 0 if not found. + * Expected format: # Latch Count: {} + * e.g. # Latch Count: 3 + */ + private int getLatchCountFromState(String theState){ + String keyStr = "# Latch Count: "; + int index = theState.indexOf(keyStr); + return index == -1 ? 0 : theState.charAt(index + keyStr.length()) - '0'; + } + + @ParameterizedTest + @ValueSource(strings = { + """ + # new code only + 1|COMPLETED + 2|COMPLETED + 2|COMPLETED + 3|GATE_WAITING,3|QUEUED + 3|GATE_WAITING,3|QUEUED + """, + """ + # OLD code only + 1|COMPLETED + 2|COMPLETED + 2|COMPLETED + 3|QUEUED,3|QUEUED + 3|QUEUED,3|QUEUED + """, + """ + # mixed code + 1|COMPLETED + 2|COMPLETED + 2|COMPLETED + 3|GATE_WAITING,3|QUEUED + 3|QUEUED,3|QUEUED + """ + }) + default void testGatedStep2ReadyToAdvance_advanceToStep3(String theChunkState) throws InterruptedException { + // setup + PointcutLatch sendingLatch = getTestManager().disableWorkChunkMessageHandler(); + sendingLatch.setExpectedCount(2); + JobMaintenanceStateInformation state = setupGatedWorkChunkTransitionTest(theChunkState, true); + getTestManager().createChunksInStates(state); + + // test + getTestManager().runMaintenancePass(); + + // verify + getTestManager().verifyWorkChunkMessageHandlerCalled(sendingLatch, 2); + assertEquals(LAST_STEP_ID, getJobInstanceFromState(state).getCurrentGatedStepId()); + verifyWorkChunkFinalStates(state); + } + + @Test + default void test_ungatedJob_queuesReadyChunks() throws InterruptedException { + // setup + String state = """ + # READY chunks should transition; others should stay + 1|COMPLETED + 2|READY,2|QUEUED + 2|READY,2|QUEUED + 2|COMPLETED + 2|IN_PROGRESS + 3|IN_PROGRESS + """; + int expectedTransitions = 2; + JobMaintenanceStateInformation result = setupGatedWorkChunkTransitionTest(state, false); + + PointcutLatch sendLatch = getTestManager().disableWorkChunkMessageHandler(); + sendLatch.setExpectedCount(expectedTransitions); + getTestManager().createChunksInStates(result); + + // TEST run job maintenance - force transition + getTestManager().enableMaintenanceRunner(true); + + getTestManager().runMaintenancePass(); + + // verify + getTestManager().verifyWorkChunkMessageHandlerCalled(sendLatch, expectedTransitions); + verifyWorkChunkFinalStates(result); + } + + private JobMaintenanceStateInformation setupGatedWorkChunkTransitionTest(String theChunkState, boolean theIsGated) { + // get the job def and store the instance + JobDefinition definition = getTestManager().withJobDefinition(theIsGated); + String instanceId = getTestManager().createAndStoreJobInstance(definition); + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation(instanceId, definition, theChunkState); + + ourLog.info("Starting test case \n {}", theChunkState); + // display comments if there are any + ourLog.info(String.join(", ", stateInformation.getLineComments())); + return stateInformation; + } + + private void verifyWorkChunkFinalStates(JobMaintenanceStateInformation theStateInformation) { + theStateInformation.verifyFinalStates(getTestManager().getSvc()); + } + + private JobInstance getJobInstanceFromState(JobMaintenanceStateInformation state) { + return getTestManager().freshFetchJobInstance(state.getInstanceId()); + } +} diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/ITestFixture.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/ITestFixture.java index fdb75990f0d..0ae2302ff40 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/ITestFixture.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/ITestFixture.java @@ -23,6 +23,7 @@ import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.hapi.fhir.batch2.test.support.JobMaintenanceStateInformation; import ca.uhn.hapi.fhir.batch2.test.support.TestJobParameters; import ca.uhn.test.concurrency.PointcutLatch; import org.springframework.transaction.PlatformTransactionManager; @@ -36,12 +37,14 @@ public interface ITestFixture { WorkChunk freshFetchWorkChunk(String theChunkId); - String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData); + String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData, boolean theGatedExecution); void runInTransaction(Runnable theRunnable); void sleepUntilTimeChanges(); + JobDefinition withJobDefinitionWithReductionStep(); + JobDefinition withJobDefinition(boolean theIsGatedJob); TransactionTemplate newTxTemplate(); @@ -61,6 +64,14 @@ public interface ITestFixture { */ String createChunk(String theJobInstanceId); + String createChunk(String theJobInstanceId, boolean theGatedExecution); + + /** + * Create chunk as the first chunk of a job. + * @return the id of the created chunk + */ + String createFirstChunk(JobDefinition theJobDefinition, String theJobInstanceId); + /** * Enable/disable the maintenance runner (So it doesn't run on a scheduler) */ @@ -81,4 +92,10 @@ public interface ITestFixture { * @param theNumberOfTimes the number of invocations to expect */ void verifyWorkChunkMessageHandlerCalled(PointcutLatch theSendingLatch, int theNumberOfTimes) throws InterruptedException; + + /** + * Uses the JobMaintenanceStateInformation to setup a test. + * @param theJobMaintenanceStateInformation + */ + void createChunksInStates(JobMaintenanceStateInformation theJobMaintenanceStateInformation); } diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkErrorActionsTests.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkErrorActionsTests.java index 092f4092599..0768ed19541 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkErrorActionsTests.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkErrorActionsTests.java @@ -31,7 +31,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public interface IWorkChunkErrorActionsTests extends IWorkChunkCommon, WorkChunkTestConstants { - /** * The consumer will retry after a retryable error is thrown */ diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStateTransitions.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStateTransitions.java index 7f906173df9..d4dff0d44d4 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStateTransitions.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStateTransitions.java @@ -19,23 +19,82 @@ */ package ca.uhn.hapi.fhir.batch2.test; +import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.hapi.fhir.batch2.test.support.JobMaintenanceStateInformation; +import ca.uhn.hapi.fhir.batch2.test.support.TestJobParameters; +import ca.uhn.test.concurrency.LockstepEnumPhaser; import ca.uhn.test.concurrency.PointcutLatch; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Nested public interface IWorkChunkStateTransitions extends IWorkChunkCommon, WorkChunkTestConstants { Logger ourLog = LoggerFactory.getLogger(IWorkChunkStateTransitions.class); - @Test - default void chunkReceived_queuedToInProgress() throws InterruptedException { + @BeforeEach + default void before() { + getTestManager().enableMaintenanceRunner(false); + } + + @ParameterizedTest + @CsvSource({ + "false, READY", + "true, GATE_WAITING" + }) + default void chunkCreation_nonFirstChunk_isInExpectedStatus(boolean theGatedExecution, WorkChunkStatusEnum expectedStatus) { String jobInstanceId = getTestManager().createAndStoreJobInstance(null); - String myChunkId = getTestManager().createChunk(jobInstanceId); + String myChunkId = getTestManager().createChunk(jobInstanceId, theGatedExecution); + + WorkChunk fetchedWorkChunk = getTestManager().freshFetchWorkChunk(myChunkId); + assertEquals(expectedStatus, fetchedWorkChunk.getStatus(), "New chunks are " + expectedStatus); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + default void chunkCreation_firstChunk_isInReady(boolean theGatedExecution) { + JobDefinition jobDef = getTestManager().withJobDefinition(theGatedExecution); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + String myChunkId = getTestManager().createFirstChunk(jobDef, jobInstanceId); + + WorkChunk fetchedWorkChunk = getTestManager().freshFetchWorkChunk(myChunkId); + // the first chunk of both gated and non-gated job should start in READY + assertEquals(WorkChunkStatusEnum.READY, fetchedWorkChunk.getStatus(), "New chunks are " + WorkChunkStatusEnum.READY); + } + + @Test + default void chunkReceived_forNongatedJob_queuedToInProgress() throws InterruptedException { + PointcutLatch sendLatch = getTestManager().disableWorkChunkMessageHandler(); + sendLatch.setExpectedCount(1); + + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + String myChunkId = getTestManager().createChunk(jobInstanceId, false); getTestManager().runMaintenancePass(); // the worker has received the chunk, and marks it started. @@ -47,5 +106,374 @@ public interface IWorkChunkStateTransitions extends IWorkChunkCommon, WorkChunkT // verify the db was updated too WorkChunk fetchedWorkChunk = getTestManager().freshFetchWorkChunk(myChunkId); assertEquals(WorkChunkStatusEnum.IN_PROGRESS, fetchedWorkChunk.getStatus()); + getTestManager().verifyWorkChunkMessageHandlerCalled(sendLatch, 1); + } + + /** + * Tests transitions to a known ready state for gated jobs. + * for most jobs, this is READY. For reduction step jobs, this is REDUCTION_READY + */ + @ParameterizedTest + @ValueSource(booleans = { true, false }) + default void advanceJobStepAndUpdateChunkStatus_forGatedJob_updatesBothGATE_WAITINGAndQUEUEDChunksToAnExpectedREADYState(boolean theIsReductionStep) { + // setup + getTestManager().disableWorkChunkMessageHandler(); + + WorkChunkStatusEnum nextState = theIsReductionStep ? WorkChunkStatusEnum.REDUCTION_READY : WorkChunkStatusEnum.READY; + String state = String.format(""" + 1|COMPLETED + 2|COMPLETED + 3|GATE_WAITING,3|%s + 3|QUEUED,3|%s + """, nextState.name(), nextState.name()); + + JobDefinition jobDef = theIsReductionStep ? getTestManager().withJobDefinitionWithReductionStep() : getTestManager().withJobDefinition(true); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation(jobInstanceId, jobDef, state); + getTestManager().createChunksInStates(info); + assertEquals(SECOND_STEP_ID, getTestManager().freshFetchJobInstance(jobInstanceId).getCurrentGatedStepId()); + + // execute + getTestManager().runInTransaction(() -> getTestManager().getSvc().advanceJobStepAndUpdateChunkStatus(jobInstanceId, LAST_STEP_ID, theIsReductionStep)); + + // verify + assertEquals(LAST_STEP_ID, getTestManager().freshFetchJobInstance(jobInstanceId).getCurrentGatedStepId()); + info.verifyFinalStates(getTestManager().getSvc()); + } + + @ParameterizedTest + @ValueSource(strings = { + """ + 1|COMPLETED + 2|COMPLETED + 3|READY + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|REDUCTION_READY + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|IN_PROGRESS + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|POLL_WAITING + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|ERRORED + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|FAILED + """, + """ + 1|COMPLETED + 2|COMPLETED + 3|COMPLETED + """ + }) + default void advanceJobStepAndUpdateChunkStatus_reductionJobWithInvalidStates_doNotTransition(String theState) { + // setup + getTestManager().disableWorkChunkMessageHandler(); + + JobDefinition jobDef = getTestManager().withJobDefinitionWithReductionStep(); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation(jobInstanceId, jobDef, theState); + getTestManager().createChunksInStates(info); + assertEquals(SECOND_STEP_ID, getTestManager().freshFetchJobInstance(jobInstanceId).getCurrentGatedStepId()); + + // execute + getTestManager().runInTransaction(() -> { + getTestManager().getSvc().advanceJobStepAndUpdateChunkStatus(jobInstanceId, LAST_STEP_ID, true); + }); + + // verify + assertEquals(LAST_STEP_ID, getTestManager().freshFetchJobInstance(jobInstanceId).getCurrentGatedStepId()); + info.verifyFinalStates(getTestManager().getSvc()); + } + + @Test + default void enqueueWorkChunkForProcessing_enqueuesOnlyREADYChunks() throws InterruptedException { + // setup + getTestManager().disableWorkChunkMessageHandler(); + + StringBuilder sb = new StringBuilder(); + // first step is always complete + sb.append("1|COMPLETED"); + for (WorkChunkStatusEnum status : WorkChunkStatusEnum.values()) { + if (!sb.isEmpty()) { + sb.append("\n"); + } + // second step for all other workchunks + sb.append("2|") + .append(status.name()); + } + String state = sb.toString(); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDef); + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation( + instanceId, + jobDef, + state + ); + getTestManager().createChunksInStates(stateInformation); + + // test + PointcutLatch latch = new PointcutLatch(new Exception().getStackTrace()[0].getMethodName()); + latch.setExpectedCount(stateInformation.getInitialWorkChunks().size()); + for (WorkChunk chunk : stateInformation.getInitialWorkChunks()) { + getTestManager().getSvc().enqueueWorkChunkForProcessing(chunk.getId(), updated -> { + // should not update non-ready chunks + ourLog.info("Enqueuing chunk with state {}; updated {}", chunk.getStatus().name(), updated); + int expected = chunk.getStatus() == WorkChunkStatusEnum.READY ? 1 : 0; + assertEquals(expected, updated); + latch.call(1); + }); + } + latch.awaitExpected(); + } + + /** + * Nasty test for a nasty bug. + * We use the transactional-outbox pattern to guarantee at-least-once delivery to the kafka queue by sending to + * kafka before the READY->QUEUED tx commits. + * BUT, kakfa is so fast, the listener may deque before the transition. Our listener is confused if the chunk + * is still in READY. + * This test uses a lock-step phaser to create this scenario, and makes sure the dequeue is lock-consistent with the enqueue. + * + */ + @Test + default void testSimultaneousDeque_beforeEnqueCommit_doesNotDropChunk() throws ExecutionException, InterruptedException, TimeoutException { + // given + PointcutLatch pointcutLatch = getTestManager().disableWorkChunkMessageHandler(); + getTestManager().enableMaintenanceRunner(false); + + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation( + jobInstanceId, + jobDef, + """ + 2|READY,2|IN_PROGRESS + """ + ); + stateInformation.initialize(getTestManager().getSvc()); + String chunkId = stateInformation.getInitialWorkChunks() + .stream().findFirst().orElseThrow().getId(); + + enum Steps { + STARTING, SENT_TO_KAFA_BEFORE_COMMIT, COMMIT_QUEUED_STATUS, FINISHED + } + LockstepEnumPhaser phaser = new LockstepEnumPhaser<>(3, Steps.class); + phaser.assertInPhase(Steps.STARTING); + + // test + ExecutorService workerThreads = Executors.newFixedThreadPool(2, new BasicThreadFactory.Builder().namingPattern("Deque-race-%d").build()); + try { + + // thread 1 - mimic the maintenance queueing a chunk notification to kafka + workerThreads.submit(()-> getTestManager().getSvc().enqueueWorkChunkForProcessing(chunkId, (i)->{ + phaser.arriveAndAwaitSharedEndOf(Steps.STARTING); + ourLog.info("Fake send chunk to kafka {}", chunkId); + phaser.arriveAndAwaitSharedEndOf(Steps.SENT_TO_KAFA_BEFORE_COMMIT); + // wait for listener to "receive" our notification + phaser.arriveAndAwaitSharedEndOf(Steps.COMMIT_QUEUED_STATUS); + // wait here. + })); + + // thread 2 - mimic the kafka listener receiving a notification before the maintenance tx has committed + Future> dequeueResult = workerThreads.submit(() -> { + phaser.arriveAndAwaitSharedEndOf(Steps.STARTING); + phaser.arriveAndAwaitSharedEndOf(Steps.SENT_TO_KAFA_BEFORE_COMMIT); + phaser.arriveAndDeregister(); + Optional workChunk = getTestManager().getSvc().onWorkChunkDequeue(chunkId); + + return workChunk; + }); + + phaser.arriveAndAwaitSharedEndOf(Steps.STARTING); + phaser.arriveAndAwaitSharedEndOf(Steps.SENT_TO_KAFA_BEFORE_COMMIT); + // wait while the deque tries to run + Thread.sleep(100); + phaser.arriveAndAwaitSharedEndOf(Steps.COMMIT_QUEUED_STATUS); + Optional workChunk = dequeueResult.get(1, TimeUnit.SECONDS); + + assertTrue(workChunk.isPresent(), "Found the chunk despite being simultaneous"); + stateInformation.verifyFinalStates(getTestManager().getSvc()); + + } finally { + workerThreads.shutdownNow(); + pointcutLatch.clear(); + + } + + } + + + @ParameterizedTest + @ValueSource(strings = { + "2|READY", + "2|QUEUED", + //"2|GATED,", // TODO - update/enable when gated status is available + "2|POLL_WAITING", + "2|ERRORED", + "2|FAILED", + "2|COMPLETED" + }) + default void onWorkChunkPollDelay_withNoInProgressChunks_doNotTransitionNorSetTime(String theState) { + // setup + getTestManager().disableWorkChunkMessageHandler(); + getTestManager().enableMaintenanceRunner(false); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + // the time we set it to + Date newTime = Date.from( + Instant.now().plus(Duration.ofSeconds(100)) + ); + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation( + jobInstanceId, + jobDef, + theState + ); + stateInformation.initialize(getTestManager().getSvc()); + + String chunkId = stateInformation.getInitialWorkChunks() + .stream().findFirst().orElseThrow().getId(); + + // test + getTestManager().getSvc().onWorkChunkPollDelay(chunkId, newTime); + + // verify + stateInformation.verifyFinalStates(getTestManager().getSvc(), chunk -> assertNull(chunk.getNextPollTime())); + } + + @Test + default void onWorkChunkPollDelay_withInProgressChunks_transitionsAndSetsNewTime() { + // setup + getTestManager().disableWorkChunkMessageHandler(); + getTestManager().enableMaintenanceRunner(false); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + // the time we set it to + Date newTime = Date.from( + Instant.now().plus(Duration.ofSeconds(100)) + ); + + String state = "2|IN_PROGRESS,2|POLL_WAITING"; + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation( + jobInstanceId, jobDef, + state + ); + stateInformation.initialize(getTestManager().getSvc()); + + String chunkId = stateInformation.getInitialWorkChunks() + .stream().findFirst().orElseThrow().getId(); + + // test + getTestManager().getSvc().onWorkChunkPollDelay(chunkId, newTime); + + // verify + stateInformation.verifyFinalStates(getTestManager().getSvc(), (chunk) -> { + // verify the time has been set + assertEquals(newTime, chunk.getNextPollTime()); + assertEquals(1, chunk.getPollAttempts()); + }); + } + + @Test + default void updatePollWaitingChunksForJobIfReady_pollWaitingChunkWithExpiredTime_transition() { + updatePollWaitingChunksForJobIfReady_POLL_WAITING_chunksTest(true); + } + + @Test + default void updatePollWaitingChunksForJobIfReady_pollWaitingChunkWithNonExpiredTime_doesNotTransition() { + updatePollWaitingChunksForJobIfReady_POLL_WAITING_chunksTest(false); + } + + private void updatePollWaitingChunksForJobIfReady_POLL_WAITING_chunksTest(boolean theDeadlineIsExpired) { + // setup + getTestManager().disableWorkChunkMessageHandler(); + getTestManager().enableMaintenanceRunner(false); + String state = "1|POLL_WAITING"; + if (theDeadlineIsExpired) { + state += ",1|READY"; + } + + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation( + jobInstanceId, + jobDef, + state + ); + Date nextPollTime = theDeadlineIsExpired ? + Date.from(Instant.now().minus(Duration.ofSeconds(10))) : Date.from(Instant.now().plus(Duration.ofSeconds(10))); + stateInformation.addWorkChunkModifier(chunk -> chunk.setNextPollTime(nextPollTime)); + stateInformation.initialize(getTestManager().getSvc()); + + // test + int updateCount = getTestManager().getSvc().updatePollWaitingChunksForJobIfReady(jobInstanceId); + + // verify + if (theDeadlineIsExpired) { + assertEquals(1, updateCount); + } else { + assertEquals(0, updateCount); + } + stateInformation.verifyFinalStates(getTestManager().getSvc()); + } + + /** + * Only POLL_WAITING chunks should be able to transition to READY via + * updatePollWaitingChunksForJobIfReady + */ + @ParameterizedTest + @ValueSource(strings = { + "2|READY", + // "2|GATED", // TODO - update/enable whenever gated status is ready + "2|QUEUED", + "2|IN_PROGRESS", + "2|ERRORED", + "2|FAILED", + "2|COMPLETED" + }) + default void updatePollWaitingChunksForJobIfReady_withNoPollWaitingChunks_doNotTransitionNorUpdateTime(String theState) { + // setup + getTestManager().disableWorkChunkMessageHandler(); + getTestManager().enableMaintenanceRunner(false); + + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String jobInstanceId = getTestManager().createAndStoreJobInstance(jobDef); + + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation(jobInstanceId, + jobDef, + theState); + stateInformation.addWorkChunkModifier((chunk) -> { + // make sure time is in the past, so we aren't testing the + // time <= now aspect + chunk.setNextPollTime( + Date.from(Instant.now().minus(Duration.ofSeconds(10))) + ); + }); + stateInformation.initialize(getTestManager().getSvc()); + + // test + int updateCount = getTestManager().getSvc().updatePollWaitingChunksForJobIfReady(jobInstanceId); + + // verify + assertEquals(0, updateCount); + stateInformation.verifyFinalStates(getTestManager().getSvc()); } } diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStorageTests.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStorageTests.java index 0174dcaabdf..392898c4332 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStorageTests.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/IWorkChunkStorageTests.java @@ -21,22 +21,250 @@ package ca.uhn.hapi.fhir.batch2.test; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.WorkChunk; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNull; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.hapi.fhir.batch2.test.support.JobMaintenanceStateInformation; +import ca.uhn.test.concurrency.PointcutLatch; +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Nested public interface IWorkChunkStorageTests extends IWorkChunkCommon, WorkChunkTestConstants { + @BeforeEach + default void before() { + getTestManager().enableMaintenanceRunner(false); + } + @Test default void testStoreAndFetchWorkChunk_NoData() { JobInstance instance = createInstance(); String instanceId = getTestManager().getSvc().storeNewInstance(instance); - String id = getTestManager().storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, null); + String id = getTestManager().storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, null, false); getTestManager().runInTransaction(() -> { WorkChunk chunk = getTestManager().freshFetchWorkChunk(id); assertNull(chunk.getData()); }); } + + @ParameterizedTest + @CsvSource({ + "false, READY", + "true, GATE_WAITING" + }) + default void testWorkChunkCreate_inExpectedStatus(boolean theGatedExecution, WorkChunkStatusEnum expectedStatus) { + JobInstance instance = createInstance(); + String instanceId = getTestManager().getSvc().storeNewInstance(instance); + + String id = getTestManager().storeWorkChunk(JOB_DEFINITION_ID, FIRST_STEP_ID, instanceId, 0, CHUNK_DATA, theGatedExecution); + assertNotNull(id); + + getTestManager().runInTransaction(() -> assertEquals(expectedStatus, getTestManager().freshFetchWorkChunk(id).getStatus())); + } + + @Test + default void testNonGatedWorkChunkInReady_IsQueuedDuringMaintenance() throws InterruptedException { + // setup + int expectedCalls = 1; + PointcutLatch sendingLatch = getTestManager().disableWorkChunkMessageHandler(); + sendingLatch.setExpectedCount(expectedCalls); + String state = "1|READY,1|QUEUED"; + JobDefinition jobDefinition = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDefinition); + JobMaintenanceStateInformation stateInformation = new JobMaintenanceStateInformation(instanceId, jobDefinition, state); + + getTestManager().createChunksInStates(stateInformation); + String id = stateInformation.getInitialWorkChunks().stream().findFirst().orElseThrow().getId(); + + // verify created in ready + getTestManager().runInTransaction(() -> assertEquals(WorkChunkStatusEnum.READY, getTestManager().freshFetchWorkChunk(id).getStatus())); + + // test + getTestManager().runMaintenancePass(); + + // verify it's in QUEUED now + stateInformation.verifyFinalStates(getTestManager().getSvc()); + getTestManager().verifyWorkChunkMessageHandlerCalled(sendingLatch, expectedCalls); + } + + @Test + default void testStoreAndFetchWorkChunk_WithData() { + // setup + getTestManager().disableWorkChunkMessageHandler(); + JobDefinition jobDefinition = getTestManager().withJobDefinition(false); + JobInstance instance = createInstance(); + String instanceId = getTestManager().getSvc().storeNewInstance(instance); + + // we're not transitioning this state; we're just checking storage of data + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation(instanceId, jobDefinition, "1|QUEUED"); + info.addWorkChunkModifier((chunk) -> { + chunk.setData(CHUNK_DATA); + }); + + getTestManager().createChunksInStates(info); + String id = info.getInitialWorkChunks().stream().findFirst().orElseThrow().getId(); + + // verify created in QUEUED + getTestManager().runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, getTestManager().freshFetchWorkChunk(id).getStatus())); + + // test; manually dequeue chunk + WorkChunk chunk = getTestManager().getSvc().onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); + + // verify + assertEquals(36, chunk.getInstanceId().length()); + assertEquals(JOB_DEFINITION_ID, chunk.getJobDefinitionId()); + assertEquals(JOB_DEF_VER, chunk.getJobDefinitionVersion()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(CHUNK_DATA, chunk.getData()); + + getTestManager().runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, getTestManager().freshFetchWorkChunk(id).getStatus())); + } + + @Test + default void testMarkChunkAsCompleted_Success() { + // setup + String state = "2|IN_PROGRESS,2|COMPLETED"; + getTestManager().disableWorkChunkMessageHandler(); + + JobDefinition jobDefinition = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDefinition); + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation(instanceId, jobDefinition, state); + info.addWorkChunkModifier(chunk -> { + chunk.setCreateTime(new Date()); + chunk.setData(CHUNK_DATA); + }); + getTestManager().createChunksInStates(info); + + String chunkId = info.getInitialWorkChunks().stream().findFirst().orElseThrow().getId(); + + // run test + getTestManager().runInTransaction(() -> getTestManager().getSvc().onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 50, 0))); + + // verify + info.verifyFinalStates(getTestManager().getSvc()); + WorkChunk entity = getTestManager().freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.COMPLETED, entity.getStatus()); + assertEquals(50, entity.getRecordsProcessed()); + assertNotNull(entity.getCreateTime()); + assertNull(entity.getData()); + } + + @Test + default void testMarkChunkAsCompleted_Error() { + // setup + String state = "1|IN_PROGRESS,1|ERRORED"; + getTestManager().disableWorkChunkMessageHandler(); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDef); + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation( + instanceId, jobDef, state + ); + getTestManager().createChunksInStates(info); + String chunkId = info.getInitialWorkChunks().stream().findFirst().orElseThrow().getId(); + + // test + WorkChunkErrorEvent request = new WorkChunkErrorEvent(chunkId, ERROR_MESSAGE_A); + getTestManager().getSvc().onWorkChunkError(request); + getTestManager().runInTransaction(() -> { + WorkChunk entity = getTestManager().freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); + assertEquals(ERROR_MESSAGE_A, entity.getErrorMessage()); + assertEquals(1, entity.getErrorCount()); + }); + + // Mark errored again + + WorkChunkErrorEvent request2 = new WorkChunkErrorEvent(chunkId, "This is an error message 2"); + getTestManager().getSvc().onWorkChunkError(request2); + getTestManager().runInTransaction(() -> { + WorkChunk entity = getTestManager().freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); + assertEquals("This is an error message 2", entity.getErrorMessage()); + assertEquals(2, entity.getErrorCount()); + }); + + List chunks = ImmutableList.copyOf(getTestManager().getSvc().fetchAllWorkChunksIterator(instanceId, true)); + assertEquals(1, chunks.size()); + assertEquals(2, chunks.get(0).getErrorCount()); + + info.verifyFinalStates(getTestManager().getSvc()); + } + + @Test + default void testMarkChunkAsCompleted_Fail() { + // setup + String state = "1|IN_PROGRESS,1|FAILED"; + getTestManager().disableWorkChunkMessageHandler(); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDef); + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation( + instanceId, jobDef, state + ); + getTestManager().createChunksInStates(info); + String chunkId = info.getInitialWorkChunks().stream().findFirst().orElseThrow().getId(); + + // test + getTestManager().getSvc().onWorkChunkFailed(chunkId, "This is an error message"); + + // verify + getTestManager().runInTransaction(() -> { + WorkChunk entity = getTestManager().freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.FAILED, entity.getStatus()); + assertEquals("This is an error message", entity.getErrorMessage()); + }); + + info.verifyFinalStates(getTestManager().getSvc()); + } + + @Test + default void markWorkChunksWithStatusAndWipeData_marksMultipleChunksWithStatus_asExpected() { + // setup + String state = """ + 1|IN_PROGRESS,1|COMPLETED + 1|ERRORED,1|COMPLETED + 1|QUEUED,1|COMPLETED + 1|IN_PROGRESS,1|COMPLETED + """; + getTestManager().disableWorkChunkMessageHandler(); + JobDefinition jobDef = getTestManager().withJobDefinition(false); + String instanceId = getTestManager().createAndStoreJobInstance(jobDef); + JobMaintenanceStateInformation info = new JobMaintenanceStateInformation( + instanceId, jobDef, state + ); + getTestManager().createChunksInStates(info); + List chunkIds = info.getInitialWorkChunks().stream().map(WorkChunk::getId) + .collect(Collectors.toList()); + + getTestManager().runInTransaction(() -> getTestManager().getSvc().markWorkChunksWithStatusAndWipeData(instanceId, chunkIds, WorkChunkStatusEnum.COMPLETED, null)); + + Iterator reducedChunks = getTestManager().getSvc().fetchAllWorkChunksIterator(instanceId, true); + + while (reducedChunks.hasNext()) { + WorkChunk reducedChunk = reducedChunks.next(); + assertTrue(chunkIds.contains(reducedChunk.getId())); + assertEquals(WorkChunkStatusEnum.COMPLETED, reducedChunk.getStatus()); + } + } } diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/WorkChunkTestConstants.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/WorkChunkTestConstants.java index 8ab0157e52b..10df1376af9 100644 --- a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/WorkChunkTestConstants.java +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/WorkChunkTestConstants.java @@ -24,7 +24,9 @@ public interface WorkChunkTestConstants { // we use a separate id for gated jobs because these job definitions might not // be cleaned up after any given test run String GATED_JOB_DEFINITION_ID = "gated_job_def_id"; - public static final String TARGET_STEP_ID = "step-id"; + public static final String FIRST_STEP_ID = "step-id"; + public static final String SECOND_STEP_ID = "2nd-step-id"; + public static final String LAST_STEP_ID = "last-step-id"; public static final String DEF_CHUNK_ID = "definition-chunkId"; public static final int JOB_DEF_VER = 1; public static final int SEQUENCE_NUMBER = 1; diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/JobMaintenanceStateInformation.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/JobMaintenanceStateInformation.java new file mode 100644 index 00000000000..d83ca179928 --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/JobMaintenanceStateInformation.java @@ -0,0 +1,269 @@ +package ca.uhn.hapi.fhir.batch2.test.support; + +import ca.uhn.fhir.batch2.api.IJobPersistence; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.JobDefinitionStep; +import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * This class is used to help set up and verify WorkChunk transitions. + * + * Creating this object requires an instanceid (of a stored instance), and jobdefinition (of stored job defintion) + * and a state string. + * + * State strings are defined as follows: + * "step-name-or-step-index,INITIAL_STATE|step-name-or-step-index,FINAL_STATE" + * + * where "step-name-or-step-index" is the name or index of a step in the provided + * JobDefinition; the step that the work chunk should start in. + * + * If no final state/step name is provided, no transition is assumed. + * + * Further, comments can be added to the state string, but must be started by a "#". + * + * Eg: + * 1,READY|1,QUEUED # will create an initial work chunk in the READY state in step 1. + * # validation will verify that this workchunk has been transitioned to QUEUED. + */ +public class JobMaintenanceStateInformation { + + private static final Logger ourLog = LoggerFactory.getLogger(JobMaintenanceStateInformation.class); + + private static final String COMMENT_PATTERN = "(#.*)$"; + + private final List myLineComments = new ArrayList<>(); + + private final List myInitialWorkChunks = new ArrayList<>(); + + private final List myFinalWorkChunk = new ArrayList<>(); + + private final JobDefinition myJobDefinition; + + private final String myInstanceId; + + private Consumer myWorkChunkModifier = (chunk) -> {}; + + public JobMaintenanceStateInformation( + String theInstanceId, + JobDefinition theJobDefinition, + String theStateUnderTest) { + myInstanceId = theInstanceId; + myJobDefinition = theJobDefinition; + + setState(theStateUnderTest); + } + + public void addWorkChunkModifier(Consumer theModifier) { + myWorkChunkModifier = theModifier; + } + + public List getLineComments() { + return myLineComments; + } + + public List getInitialWorkChunks() { + return myInitialWorkChunks; + } + + public List getFinalWorkChunk() { + return myFinalWorkChunk; + } + + public JobDefinition getJobDefinition() { + return myJobDefinition; + } + + public String getInstanceId() { + return myInstanceId; + } + + public void verifyFinalStates(IJobPersistence theJobPersistence) { + verifyFinalStates(theJobPersistence, (chunk) -> {}); + } + + public void verifyFinalStates(IJobPersistence theJobPersistence, Consumer theChunkConsumer) { + assertEquals(getInitialWorkChunks().size(), getFinalWorkChunk().size()); + + HashMap workchunkMap = new HashMap<>(); + for (WorkChunk fs : getFinalWorkChunk()) { + workchunkMap.put(fs.getId(), fs); + } + + // fetch all workchunks + Iterator workChunkIterator = theJobPersistence.fetchAllWorkChunksIterator(getInstanceId(), true); + List workchunks = new ArrayList<>(); + workChunkIterator.forEachRemaining(workchunks::add); + + assertEquals(workchunks.size(), workchunkMap.size()); + workchunks.forEach(c -> ourLog.info("Returned " + c.toString())); + + for (WorkChunk wc : workchunks) { + WorkChunk expected = workchunkMap.get(wc.getId()); + assertNotNull(expected); + + // verify status and step id + assertEquals(expected.getTargetStepId(), wc.getTargetStepId()); + assertEquals(expected.getStatus(), wc.getStatus()); + theChunkConsumer.accept(wc); + } + } + + public void initialize(IJobPersistence theJobPersistence) { + // should have as many input workchunks as output workchunks + // unless we have newly created ones somewhere + assertEquals(getInitialWorkChunks().size(), getFinalWorkChunk().size()); + + Set stepIds = new HashSet<>(); + for (int i = 0; i < getInitialWorkChunks().size(); i++) { + WorkChunk workChunk = getInitialWorkChunks().get(i); + myWorkChunkModifier.accept(workChunk); + WorkChunk saved = theJobPersistence.createWorkChunk(workChunk); + ourLog.info("Created WorkChunk: " + saved.toString()); + workChunk.setId(saved.getId()); + + getFinalWorkChunk().get(i) + .setId(saved.getId()); + + stepIds.add(workChunk.getTargetStepId()); + } + // if it's a gated job, we'll manually set the step id for the instance + JobDefinition jobDef = getJobDefinition(); + if (jobDef.isGatedExecution()) { + AtomicReference latestStepId = new AtomicReference<>(); + int totalSteps = jobDef.getSteps().size(); + // ignore the last step since tests in gated jobs needs the current step to be the second-last step + for (int i = totalSteps - 2; i >= 0; i--) { + JobDefinitionStep step = jobDef.getSteps().get(i); + if (stepIds.contains(step.getStepId())) { + latestStepId.set(step.getStepId()); + break; + } + } + // should def have a value + assertNotNull(latestStepId.get()); + String instanceId = getInstanceId(); + theJobPersistence.updateInstance(instanceId, instance -> { + instance.setCurrentGatedStepId(latestStepId.get()); + return true; + }); + } + } + + private void setState(String theState) { + String[] chunkLines = theState.split("\n"); + Pattern pattern = Pattern.compile(COMMENT_PATTERN); + for (String chunkLine : chunkLines) { + String line = chunkLine.trim(); + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + String comment = matcher.group(0); + line = line.replaceAll(comment, ""); + if (isEmpty(line)) { + myLineComments.add(line); + continue; + } + // else - inline comment: eg: 1|Complete # comment + } + + addWorkChunkStates(line); + } + } + + /** + * Parses the line according to: + * (work chunk step id)|(workchunk initial state)|optionally:(work chunk final state) + */ + private void addWorkChunkStates(String theLine) { + if (theLine.contains(",")) { + // has final state + String[] states = theLine.split(","); + + int len = states.length; + if (len != 2) { + throw new RuntimeException("Unexpected number of state transitions. Expected 2, found " + states.length); + } + + addWorkChunkBasedOnState(states[0], chunk -> { + myInitialWorkChunks.add(chunk); + }); + addWorkChunkBasedOnState(states[1], chunk -> { + myFinalWorkChunk.add(chunk); + }); + } else { + // does not have final state; no change + addWorkChunkBasedOnState(theLine, chunk -> { + myInitialWorkChunks.add(chunk); + myFinalWorkChunk.add(chunk); + }); + } + } + + private void addWorkChunkBasedOnState(String theLine, Consumer theAdder) { + String[] parts = theLine.split("\\|"); + int len = parts.length; + if (len < 2) { + throw new RuntimeException("Unable to parse line " + theLine + " into initial and final states"); + } + + String stepId = getJobStepId(parts[0]); + + WorkChunkStatusEnum initialStatus = WorkChunkStatusEnum.valueOf(parts[1].trim()); + WorkChunk chunk = createBaseWorkChunk(); + chunk.setStatus(initialStatus); + chunk.setTargetStepId(stepId); + theAdder.accept(chunk); + } + + private String getJobStepId(String theIndexId) { + try { + // -1 because code is 0 indexed, but people think in 1 indexed + int index = Integer.parseInt(theIndexId.trim()) - 1; + + if (index >= myJobDefinition.getSteps().size()) { + throw new RuntimeException("Unable to find step with index " + index); + } + + int counter = 0; + for (JobDefinitionStep step : myJobDefinition.getSteps()) { + if (counter == index) { + return step.getStepId(); + } + counter++; + } + + // will never happen + throw new RuntimeException("Could not find step for index " + theIndexId); + } catch (NumberFormatException ex) { + ourLog.info("Encountered non-number {}; This will be treated as the step id itself", theIndexId); + return theIndexId; + } + } + + private WorkChunk createBaseWorkChunk() { + WorkChunk chunk = new WorkChunk(); + chunk.setJobDefinitionId(myJobDefinition.getJobDefinitionId()); + chunk.setInstanceId(myInstanceId); + chunk.setJobDefinitionVersion(myJobDefinition.getJobDefinitionVersion()); + chunk.setCreateTime(new Date()); + return chunk; + } +} diff --git a/hapi-fhir-storage-batch2/pom.xml b/hapi-fhir-storage-batch2/pom.xml index 41c53b9e27e..5d2047f4996 100644 --- a/hapi-fhir-storage-batch2/pom.xml +++ b/hapi-fhir-storage-batch2/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java index 3807e3e136f..196b94ccec3 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java @@ -25,7 +25,10 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; +import ca.uhn.fhir.batch2.model.WorkChunkMetadata; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; +import com.google.common.annotations.VisibleForTesting; import jakarta.annotation.Nonnull; import org.apache.commons.lang3.builder.ToStringBuilder; import org.slf4j.Logger; @@ -41,6 +44,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Stream; /** @@ -86,6 +90,18 @@ public interface IJobPersistence extends IWorkChunkPersistence { // on implementations @Transactional(propagation = Propagation.REQUIRES_NEW) List fetchInstances(int thePageSize, int thePageIndex); + @Transactional(propagation = Propagation.REQUIRES_NEW) + void enqueueWorkChunkForProcessing(String theChunkId, Consumer theCallback); + + /** + * Updates all Work Chunks in POLL_WAITING if their nextPollTime <= now + * for the given Job Instance. + * @param theInstanceId the instance id + * @return the number of updated chunks + */ + @Transactional + int updatePollWaitingChunksForJobIfReady(String theInstanceId); + /** * Fetch instances ordered by myCreateTime DESC */ @@ -112,8 +128,12 @@ public interface IJobPersistence extends IWorkChunkPersistence { // on implementations @Transactional(propagation = Propagation.REQUIRES_NEW) Page fetchJobInstances(JobInstanceFetchRequest theRequest); - // on implementations @Transactional(propagation = Propagation.REQUIRES_NEW) - boolean canAdvanceInstanceToNextStep(String theInstanceId, String theCurrentStepId); + /** + * Returns set of all distinct states for the specified job instance id + * and step id. + */ + @Transactional + Set getDistinctWorkChunkStatesForJobAndStep(String theInstanceId, String theCurrentStepId); /** * Fetch all chunks for a given instance. @@ -131,6 +151,16 @@ public interface IJobPersistence extends IWorkChunkPersistence { */ Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); + /** + * Fetches an iterator that retrieves WorkChunkMetadata from the db. + * @param theInstanceId instance id of job of interest + * @param theStates states of interset + * @return an iterator for the workchunks + */ + @Transactional(propagation = Propagation.SUPPORTS) + Page fetchAllWorkChunkMetadataForJobInStates( + Pageable thePageable, String theInstanceId, Set theStates); + /** * Callback to update a JobInstance within a locked transaction. * Return true from the callback if the record write should continue, or false if @@ -256,4 +286,18 @@ public interface IJobPersistence extends IWorkChunkPersistence { return markInstanceAsStatusWhenStatusIn( theJobInstanceId, StatusEnum.IN_PROGRESS, Collections.singleton(StatusEnum.QUEUED)); } + + @VisibleForTesting + WorkChunk createWorkChunk(WorkChunk theWorkChunk); + + /** + * Atomically advance the given job to the given step and change the status of all QUEUED and GATE_WAITING chunks + * in the next step to READY + * @param theJobInstanceId the id of the job instance to be updated + * @param theNextStepId the id of the next job step + * @return whether any changes were made + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + boolean advanceJobStepAndUpdateChunkStatus( + String theJobInstanceId, String theNextStepId, boolean theIsReductionStepBoolean); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java index 0bde5e4d524..1bfd5c9d62b 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java @@ -27,6 +27,7 @@ import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -55,7 +56,8 @@ public interface IWorkChunkPersistence { * The first state event, as the chunk is created. * This method should be atomic and should only * return when the chunk has been successfully stored in the database. - * Chunk should be stored with a status of {@link WorkChunkStatusEnum#QUEUED} + * Chunk should be stored with a status of {@link WorkChunkStatusEnum#READY} or + * {@link WorkChunkStatusEnum#GATE_WAITING} for ungated and gated jobs, respectively. * * @param theBatchWorkChunk the batch work chunk to be stored * @return a globally unique identifier for this chunk. @@ -84,6 +86,17 @@ public interface IWorkChunkPersistence { // on impl - @Transactional(propagation = Propagation.REQUIRES_NEW) WorkChunkStatusEnum onWorkChunkError(WorkChunkErrorEvent theParameters); + /** + * Updates the specified Work Chunk to set the next polling interval. + * It wil also: + * * update the poll attempts + * * sets the workchunk status to POLL_WAITING (if it's not already in this state) + * @param theChunkId the id of the chunk to update + * @param theNewDeadline the time when polling should be redone + */ + @Transactional + void onWorkChunkPollDelay(String theChunkId, Date theNewDeadline); + /** * An unrecoverable error. * Transition to {@link WorkChunkStatusEnum#FAILED} @@ -130,14 +143,4 @@ public interface IWorkChunkPersistence { */ @Transactional(propagation = Propagation.MANDATORY, readOnly = true) Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); - - /** - * Fetch chunk ids for starting a gated step. - * - * @param theInstanceId the job - * @param theStepId the step that is starting - * @return the WorkChunk ids - */ - List fetchAllChunkIdsForStepWithStatus( - String theInstanceId, String theStepId, WorkChunkStatusEnum theStatusEnum); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/RetryChunkLaterException.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/RetryChunkLaterException.java new file mode 100644 index 00000000000..33217b87f66 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/RetryChunkLaterException.java @@ -0,0 +1,33 @@ +package ca.uhn.fhir.batch2.api; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; + +/** + * Exception that is thrown when a polling step needs to be retried at a later + * time. + */ +public class RetryChunkLaterException extends RuntimeException { + + private static final Duration ONE_MINUTE = Duration.of(1, ChronoUnit.MINUTES); + + /** + * The delay to wait (in ms) for the next poll call. + * For now, it's a constant, but we hold it here in + * case we want to change this behaviour in the future. + */ + private final Duration myNextPollDuration; + + public RetryChunkLaterException() { + this(ONE_MINUTE); + } + + public RetryChunkLaterException(Duration theDuration) { + super(); + this.myNextPollDuration = theDuration; + } + + public Duration getNextPollDuration() { + return myNextPollDuration; + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java index d7d13c624da..751b096f8a1 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java @@ -28,7 +28,6 @@ import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; -import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.i18n.Msg; @@ -48,8 +47,6 @@ import org.slf4j.Logger; import org.springframework.data.domain.Page; import org.springframework.messaging.MessageHandler; import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.support.TransactionSynchronization; -import org.springframework.transaction.support.TransactionSynchronizationManager; import java.util.Arrays; import java.util.HashSet; @@ -143,44 +140,18 @@ public class JobCoordinatorImpl implements IJobCoordinator { myJobParameterJsonValidator.validateJobParameters(theRequestDetails, theStartRequest, jobDefinition); + // we only create the first chunk amd job here + // JobMaintenanceServiceImpl.doMaintenancePass will handle the rest IJobPersistence.CreateResult instanceAndFirstChunk = myTransactionService .withSystemRequestOnDefaultPartition() .withPropagation(Propagation.REQUIRES_NEW) .execute(() -> myJobPersistence.onCreateWithFirstChunk(jobDefinition, theStartRequest.getParameters())); - JobWorkNotification workNotification = JobWorkNotification.firstStepNotification( - jobDefinition, instanceAndFirstChunk.jobInstanceId, instanceAndFirstChunk.workChunkId); - sendBatchJobWorkNotificationAfterCommit(workNotification); - Batch2JobStartResponse response = new Batch2JobStartResponse(); response.setInstanceId(instanceAndFirstChunk.jobInstanceId); return response; } - /** - * In order to make sure that the data is actually in the DB when JobWorkNotification is handled, - * this method registers a transaction synchronization that sends JobWorkNotification to Job WorkChannel - * if and when the current database transaction is successfully committed. - * If the transaction is rolled back, the JobWorkNotification will not be sent to the job WorkChannel. - */ - private void sendBatchJobWorkNotificationAfterCommit(final JobWorkNotification theJobWorkNotification) { - if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public int getOrder() { - return 0; - } - - @Override - public void afterCommit() { - myBatchJobSender.sendWorkChannelMessage(theJobWorkNotification); - } - }); - } else { - myBatchJobSender.sendWorkChannelMessage(theJobWorkNotification); - } - } - /** * Cache will be used if an identical job is QUEUED or IN_PROGRESS. Otherwise a new one will kickoff. */ diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java index 525db23a96f..0c067ef7a7c 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java @@ -50,9 +50,10 @@ class JobDataSink myTargetStep; private final AtomicInteger myChunkCounter = new AtomicInteger(0); private final AtomicReference myLastChunkId = new AtomicReference<>(); - private final boolean myGatedExecution; private final IHapiTransactionService myHapiTransactionService; + private final boolean myGatedExecution; + JobDataSink( @Nonnull BatchJobSender theBatchJobSender, @Nonnull IJobPersistence theJobPersistence, @@ -66,8 +67,8 @@ class JobDataSink { + if (updated == 1) { + JobWorkNotification workNotification = new JobWorkNotification( + myJobDefinitionId, myJobDefinitionVersion, instanceId, targetStepId, chunkId); + myBatchJobSender.sendWorkChannelMessage(workNotification); + } else { + ourLog.error( + "Expected to have updated 1 workchunk, but instead found {}. Chunk is not sent to queue.", + updated); + } + }); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java index a39dccb9a75..7d1026841dd 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java @@ -75,6 +75,12 @@ public class JobStepExecutor { myJobPersistence.updateInstance(instance.getInstanceId(), theInstance -> { @@ -337,10 +337,13 @@ public class ReductionStepExecutorServiceImpl implements IReductionStepExecutorS ReductionStepChunkProcessingResponse theResponseObject, JobWorkCursor theJobWorkCursor) { - if (!theChunk.getStatus().isIncomplete()) { + /* + * Reduction steps are done inline and only on gated jobs. + */ + if (theChunk.getStatus() == WorkChunkStatusEnum.COMPLETED) { // This should never happen since jobs with reduction are required to be gated ourLog.error( - "Unexpected chunk {} with status {} found while reducing {}. No chunks feeding into a reduction step should be complete.", + "Unexpected chunk {} with status {} found while reducing {}. No chunks feeding into a reduction step should be in a state other than READY.", theChunk.getId(), theChunk.getStatus(), theInstance); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java index 2a599d0e6d9..a092c55ce2a 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java @@ -23,6 +23,7 @@ import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.IJobStepWorker; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.JobStepFailedException; +import ca.uhn.fhir.batch2.api.RetryChunkLaterException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; @@ -34,6 +35,10 @@ import ca.uhn.fhir.util.Logs; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + public class StepExecutor { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); private final IJobPersistence myJobPersistence; @@ -57,6 +62,14 @@ public class StepExecutor { try { outcome = theStepWorker.run(theStepExecutionDetails, theDataSink); Validate.notNull(outcome, "Step theWorker returned null: %s", theStepWorker.getClass()); + } catch (RetryChunkLaterException ex) { + Date nextPollTime = Date.from(Instant.now().plus(ex.getNextPollDuration())); + ourLog.debug( + "Polling job encountered; will retry chunk {} after after {}s", + theStepExecutionDetails.getChunkId(), + ex.getNextPollDuration().get(ChronoUnit.SECONDS)); + myJobPersistence.onWorkChunkPollDelay(theStepExecutionDetails.getChunkId(), nextPollTime); + return false; } catch (JobExecutionFailedException e) { ourLog.error( "Unrecoverable failure executing job {} step {} chunk {}", diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java index b5cacf9fedf..7edff129858 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java @@ -251,8 +251,10 @@ class WorkChannelMessageHandler implements MessageHandler { processingPreparation.ifPresentOrElse( // all the setup is happy and committed. Do the work. process -> process.myStepExector.executeStep(), - // discard the chunk - () -> ourLog.debug("Discarding chunk notification {}", workNotification)); + () -> { + // discard the chunk + ourLog.debug("Discarding chunk notification {}", workNotification); + }); } /** diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java index 32a3ca79cb9..e08de1fb277 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java @@ -28,21 +28,30 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkMetadata; import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.batch2.progress.JobInstanceProgressCalculator; import ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.model.api.PagingIterator; import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.util.StopWatch; import org.apache.commons.lang3.time.DateUtils; import org.slf4j.Logger; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import java.util.Iterator; import java.util.List; +import java.util.Optional; +import java.util.Set; public class JobInstanceProcessor { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); public static final long PURGE_THRESHOLD = 7L * DateUtils.MILLIS_PER_DAY; + // 10k; we want to get as many as we can + private static final int WORK_CHUNK_METADATA_BATCH_SIZE = 10000; private final IJobPersistence myJobPersistence; private final BatchJobSender myBatchJobSender; private final JobChunkProgressAccumulator myProgressAccumulator; @@ -84,8 +93,38 @@ public class JobInstanceProcessor { // reload after update theInstance = myJobPersistence.fetchInstance(myInstanceId).orElseThrow(); } + + JobDefinition jobDefinition = + myJobDefinitionegistry.getJobDefinitionOrThrowException(theInstance); + + // move POLL_WAITING -> READY + processPollingChunks(theInstance.getInstanceId()); + // determine job progress; delete CANCELED/COMPLETE/FAILED jobs that are no longer needed cleanupInstance(theInstance); - triggerGatedExecutions(theInstance); + // move gated jobs to the next step, if needed + // moves GATE_WAITING / QUEUED (legacy) chunks to: + // READY (for regular gated jobs) + // REDUCTION_READY (if it's the final reduction step) + triggerGatedExecutions(theInstance, jobDefinition); + + if (theInstance.hasGatedStep() && theInstance.isRunning()) { + Optional updatedInstance = myJobPersistence.fetchInstance(theInstance.getInstanceId()); + + if (updatedInstance.isEmpty()) { + return; + } + + JobWorkCursor jobWorkCursor = JobWorkCursor.fromJobDefinitionAndRequestedStepId( + jobDefinition, updatedInstance.get().getCurrentGatedStepId()); + if (jobWorkCursor.isReductionStep()) { + // Reduction step work chunks should never be sent to the queue but to its specific service instead. + triggerReductionStep(theInstance, jobWorkCursor); + return; + } + } + + // enqueue all READY chunks + enqueueReadyChunks(theInstance, jobDefinition); ourLog.debug("Finished job processing: {} - {}", myInstanceId, stopWatch); } @@ -156,8 +195,10 @@ public class JobInstanceProcessor { return false; } - private void triggerGatedExecutions(JobInstance theInstance) { - if (!theInstance.isRunning()) { + private void triggerGatedExecutions(JobInstance theInstance, JobDefinition theJobDefinition) { + // QUEUE'd jobs that are gated need to start; this step will do that + if (!theInstance.isRunning() + && (theInstance.getStatus() != StatusEnum.QUEUED && theJobDefinition.isGatedExecution())) { ourLog.debug( "JobInstance {} is not in a \"running\" state. Status {}", theInstance.getInstanceId(), @@ -169,89 +210,183 @@ public class JobInstanceProcessor { return; } - JobDefinition jobDefinition = - myJobDefinitionegistry.getJobDefinitionOrThrowException(theInstance); - JobWorkCursor jobWorkCursor = - JobWorkCursor.fromJobDefinitionAndRequestedStepId(jobDefinition, theInstance.getCurrentGatedStepId()); - - // final step - if (jobWorkCursor.isFinalStep() && !jobWorkCursor.isReductionStep()) { - ourLog.debug("Job instance {} is in final step and it's not a reducer step", theInstance.getInstanceId()); - return; - } - + JobWorkCursor jobWorkCursor = JobWorkCursor.fromJobDefinitionAndRequestedStepId( + theJobDefinition, theInstance.getCurrentGatedStepId()); String instanceId = theInstance.getInstanceId(); String currentStepId = jobWorkCursor.getCurrentStepId(); - boolean shouldAdvance = myJobPersistence.canAdvanceInstanceToNextStep(instanceId, currentStepId); - if (shouldAdvance) { - String nextStepId = jobWorkCursor.nextStep.getStepId(); - ourLog.info( - "All processing is complete for gated execution of instance {} step {}. Proceeding to step {}", - instanceId, - currentStepId, - nextStepId); + boolean canAdvance = canAdvanceGatedJob(theInstance); + if (canAdvance) { + if (!jobWorkCursor.isFinalStep()) { + // all other gated job steps except for final steps - final steps does not need to be advanced + String nextStepId = jobWorkCursor.nextStep.getStepId(); + ourLog.info( + "All processing is complete for gated execution of instance {} step {}. Proceeding to step {}", + instanceId, + currentStepId, + nextStepId); - if (jobWorkCursor.nextStep.isReductionStep()) { - JobWorkCursor nextJobWorkCursor = JobWorkCursor.fromJobDefinitionAndRequestedStepId( - jobDefinition, jobWorkCursor.nextStep.getStepId()); - myReductionStepExecutorService.triggerReductionStep(instanceId, nextJobWorkCursor); + processChunksForNextGatedSteps(theInstance, theJobDefinition, nextStepId); } else { - // otherwise, continue processing as expected - processChunksForNextSteps(theInstance, nextStepId); + ourLog.info( + "Ready to advance gated execution of instance {} but already at the final step {}. Not proceeding to advance steps.", + instanceId, + jobWorkCursor.getCurrentStepId()); } } else { + String stepId = jobWorkCursor.nextStep != null + ? jobWorkCursor.nextStep.getStepId() + : jobWorkCursor.getCurrentStepId(); ourLog.debug( "Not ready to advance gated execution of instance {} from step {} to {}.", instanceId, currentStepId, - jobWorkCursor.nextStep.getStepId()); + stepId); } } - private void processChunksForNextSteps(JobInstance theInstance, String nextStepId) { + private boolean canAdvanceGatedJob(JobInstance theInstance) { + // make sure our instance still exists + if (myJobPersistence.fetchInstance(theInstance.getInstanceId()).isEmpty()) { + // no more job + return false; + } + String currentGatedStepId = theInstance.getCurrentGatedStepId(); + + Set workChunkStatuses = myJobPersistence.getDistinctWorkChunkStatesForJobAndStep( + theInstance.getInstanceId(), currentGatedStepId); + + if (workChunkStatuses.isEmpty()) { + // no work chunks = no output + // trivial to advance to next step + ourLog.info("No workchunks for {} in step id {}", theInstance.getInstanceId(), currentGatedStepId); + return true; + } + + // all workchunks for the current step are in COMPLETED -> proceed. + return workChunkStatuses.equals(Set.of(WorkChunkStatusEnum.COMPLETED)); + } + + protected PagingIterator getReadyChunks() { + return new PagingIterator<>(WORK_CHUNK_METADATA_BATCH_SIZE, (index, batchsize, consumer) -> { + Pageable pageable = Pageable.ofSize(batchsize).withPage(index); + Page results = myJobPersistence.fetchAllWorkChunkMetadataForJobInStates( + pageable, myInstanceId, Set.of(WorkChunkStatusEnum.READY)); + for (WorkChunkMetadata metadata : results) { + consumer.accept(metadata); + } + }); + } + + /** + * Trigger the reduction step for the given job instance. Reduction step chunks should never be queued. + */ + private void triggerReductionStep(JobInstance theInstance, JobWorkCursor jobWorkCursor) { String instanceId = theInstance.getInstanceId(); - List queuedChunksForNextStep = - myProgressAccumulator.getChunkIdsWithStatus(instanceId, nextStepId, WorkChunkStatusEnum.QUEUED); + ourLog.debug("Triggering Reduction step {} of instance {}.", jobWorkCursor.getCurrentStepId(), instanceId); + myReductionStepExecutorService.triggerReductionStep(instanceId, jobWorkCursor); + } + + /** + * Chunks are initially created in READY state. + * We will move READY chunks to QUEUE'd and send them to the queue/topic (kafka) + * for processing. + */ + private void enqueueReadyChunks(JobInstance theJobInstance, JobDefinition theJobDefinition) { + Iterator iter = getReadyChunks(); + + int counter = 0; + while (iter.hasNext()) { + WorkChunkMetadata metadata = iter.next(); + + /* + * For each chunk id + * * Move to QUEUE'd + * * Send to topic + * * flush changes + * * commit + */ + updateChunkAndSendToQueue(metadata); + counter++; + } + ourLog.debug( + "Encountered {} READY work chunks for job {} of type {}", + counter, + theJobInstance.getInstanceId(), + theJobDefinition.getJobDefinitionId()); + } + + /** + * Updates the Work Chunk and sends it to the queue. + * + * Because ReductionSteps are done inline by the maintenance pass, + * those will not be sent to the queue (but they will still have their + * status updated from READY -> QUEUED). + * + * Returns true after processing. + */ + private void updateChunkAndSendToQueue(WorkChunkMetadata theChunk) { + String chunkId = theChunk.getId(); + myJobPersistence.enqueueWorkChunkForProcessing(chunkId, updated -> { + ourLog.info("Updated {} workchunk with id {}", updated, chunkId); + if (updated == 1) { + sendNotification(theChunk); + } else { + // means the work chunk is likely already gone... + // we'll log and skip it. If it's still in the DB, the next pass + // will pick it up. Otherwise, it's no longer important + ourLog.error( + "Job Instance {} failed to transition work chunk with id {} from READY to QUEUED; found {}, expected 1; skipping work chunk.", + theChunk.getInstanceId(), + theChunk.getId(), + updated); + } + }); + } + + private void sendNotification(WorkChunkMetadata theChunk) { + // send to the queue + // we use current step id because it has not been moved to the next step (yet) + JobWorkNotification workNotification = new JobWorkNotification( + theChunk.getJobDefinitionId(), + theChunk.getJobDefinitionVersion(), + theChunk.getInstanceId(), + theChunk.getTargetStepId(), + theChunk.getId()); + myBatchJobSender.sendWorkChannelMessage(workNotification); + } + + private void processChunksForNextGatedSteps( + JobInstance theInstance, JobDefinition theJobDefinition, String nextStepId) { + String instanceId = theInstance.getInstanceId(); + + List gateWaitingChunksForNextStep = myProgressAccumulator.getChunkIdsWithStatus( + instanceId, nextStepId, WorkChunkStatusEnum.GATE_WAITING, WorkChunkStatusEnum.QUEUED); int totalChunksForNextStep = myProgressAccumulator.getTotalChunkCountForInstanceAndStep(instanceId, nextStepId); - if (totalChunksForNextStep != queuedChunksForNextStep.size()) { + if (totalChunksForNextStep != gateWaitingChunksForNextStep.size()) { ourLog.debug( - "Total ProgressAccumulator QUEUED chunk count does not match QUEUED chunk size! [instanceId={}, stepId={}, totalChunks={}, queuedChunks={}]", + "Total ProgressAccumulator GATE_WAITING chunk count does not match GATE_WAITING chunk size! [instanceId={}, stepId={}, totalChunks={}, queuedChunks={}]", instanceId, nextStepId, totalChunksForNextStep, - queuedChunksForNextStep.size()); - } - // Note on sequence: we don't have XA transactions, and are talking to two stores (JPA + Queue) - // Sequence: 1 - So we run the query to minimize the work overlapping. - List chunksToSubmit = - myJobPersistence.fetchAllChunkIdsForStepWithStatus(instanceId, nextStepId, WorkChunkStatusEnum.QUEUED); - // Sequence: 2 - update the job step so the workers will process them. - boolean changed = myJobPersistence.updateInstance(instanceId, instance -> { - if (instance.getCurrentGatedStepId().equals(nextStepId)) { - // someone else beat us here. No changes - return false; - } - instance.setCurrentGatedStepId(nextStepId); - return true; - }); - if (!changed) { - // we collided with another maintenance job. - ourLog.warn("Skipping gate advance to {} for instance {} - already advanced.", nextStepId, instanceId); - return; + gateWaitingChunksForNextStep.size()); } - // DESIGN GAP: if we die here, these chunks will never be queued. - // Need a WAITING stage before QUEUED for chunks, so we can catch them. + JobWorkCursor jobWorkCursor = + JobWorkCursor.fromJobDefinitionAndRequestedStepId(theJobDefinition, nextStepId); + + // update the job step so the workers will process them. + // Sets all chunks from QUEUED/GATE_WAITING -> READY (REDUCTION_READY for reduction jobs) + myJobPersistence.advanceJobStepAndUpdateChunkStatus(instanceId, nextStepId, jobWorkCursor.isReductionStep()); + } + + /** + * Moves all POLL_WAITING work chunks to READY for work chunks whose + * nextPollTime has expired. + */ + private void processPollingChunks(String theInstanceId) { + int updatedChunkCount = myJobPersistence.updatePollWaitingChunksForJobIfReady(theInstanceId); - // Sequence: 3 - send the notifications - for (String nextChunkId : chunksToSubmit) { - JobWorkNotification workNotification = new JobWorkNotification(theInstance, nextStepId, nextChunkId); - myBatchJobSender.sendWorkChannelMessage(workNotification); - } ourLog.debug( - "Submitted a batch of chunks for processing. [chunkCount={}, instanceId={}, stepId={}]", - chunksToSubmit.size(), - instanceId, - nextStepId); + "Moved {} Work Chunks in POLL_WAITING to READY for Job Instance {}", updatedChunkCount, theInstanceId); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobDefinition.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobDefinition.java index d10ca861a3f..43c6d8387cb 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobDefinition.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobDefinition.java @@ -145,6 +145,13 @@ public class JobDefinition { return myGatedExecution; } + public JobDefinitionStep getStepById(String theId) { + return getSteps().stream() + .filter(s -> s.getStepId().equals(theId)) + .findFirst() + .orElse(null); + } + public boolean isLastStepReduction() { int stepCount = getSteps().size(); return stepCount >= 1 && getSteps().get(stepCount - 1).isReductionStep(); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobWorkCursor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobWorkCursor.java index 94db85defaf..687127caad1 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobWorkCursor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobWorkCursor.java @@ -104,6 +104,10 @@ public class JobWorkCursor= 1); - myJobDefinitionVersion = theJobDefinitionVersion; - return this; - } - - public String getTargetStepId() { - return myTargetStepId; - } - - public WorkChunk setTargetStepId(String theTargetStepId) { - Validate.notBlank(theTargetStepId); - myTargetStepId = theTargetStepId; - return this; - } - public String getData() { return myData; } @@ -199,33 +180,6 @@ public class WorkChunk implements IModelJson { return JsonUtil.deserialize(getData(), theType); } - public String getInstanceId() { - return myInstanceId; - } - - public WorkChunk setInstanceId(String theInstanceId) { - myInstanceId = theInstanceId; - return this; - } - - public String getId() { - return myId; - } - - public WorkChunk setId(String theId) { - Validate.notBlank(theId); - myId = theId; - return this; - } - - public int getSequence() { - return mySequence; - } - - public void setSequence(int theSequence) { - mySequence = theSequence; - } - public Date getCreateTime() { return myCreateTime; } @@ -251,6 +205,22 @@ public class WorkChunk implements IModelJson { myUpdateTime = theUpdateTime; } + public Date getNextPollTime() { + return myNextPollTime; + } + + public void setNextPollTime(Date theNextPollTime) { + myNextPollTime = theNextPollTime; + } + + public int getPollAttempts() { + return myPollAttempts; + } + + public void setPollAttempts(int thePollAttempts) { + myPollAttempts = thePollAttempts; + } + public String getWarningMessage() { return myWarningMessage; } @@ -263,19 +233,23 @@ public class WorkChunk implements IModelJson { @Override public String toString() { ToStringBuilder b = new ToStringBuilder(this); - b.append("Id", myId); - b.append("Sequence", mySequence); - b.append("Status", myStatus); - b.append("JobDefinitionId", myJobDefinitionId); - b.append("JobDefinitionVersion", myJobDefinitionVersion); - b.append("TargetStepId", myTargetStepId); - b.append("InstanceId", myInstanceId); + b.append("Id", getId()); + b.append("Sequence", getSequence()); + b.append("Status", getStatus()); + b.append("JobDefinitionId", getJobDefinitionId()); + b.append("JobDefinitionVersion", getJobDefinitionVersion()); + b.append("TargetStepId", getTargetStepId()); + b.append("InstanceId", getInstanceId()); b.append("Data", isNotBlank(myData) ? "(present)" : "(absent)"); b.append("CreateTime", myCreateTime); b.append("StartTime", myStartTime); b.append("EndTime", myEndTime); b.append("UpdateTime", myUpdateTime); b.append("RecordsProcessed", myRecordsProcessed); + if (myNextPollTime != null) { + b.append("NextPollTime", myNextPollTime); + } + b.append("PollAttempts", myPollAttempts); if (isNotBlank(myErrorMessage)) { b.append("ErrorMessage", myErrorMessage); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java index 95e07c87761..c2b489016b7 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java @@ -36,6 +36,7 @@ public class WorkChunkCreateEvent { public final String instanceId; public final int sequence; public final String serializedData; + public final boolean isGatedExecution; /** * Constructor @@ -52,20 +53,28 @@ public class WorkChunkCreateEvent { @Nonnull String theTargetStepId, @Nonnull String theInstanceId, int theSequence, - @Nullable String theSerializedData) { + @Nullable String theSerializedData, + boolean theGatedExecution) { jobDefinitionId = theJobDefinitionId; jobDefinitionVersion = theJobDefinitionVersion; targetStepId = theTargetStepId; instanceId = theInstanceId; sequence = theSequence; serializedData = theSerializedData; + isGatedExecution = theGatedExecution; } + /** + * Creates the WorkChunkCreateEvent for the first chunk of a job. + */ public static WorkChunkCreateEvent firstChunk(JobDefinition theJobDefinition, String theInstanceId) { String firstStepId = theJobDefinition.getFirstStepId(); String jobDefinitionId = theJobDefinition.getJobDefinitionId(); int jobDefinitionVersion = theJobDefinition.getJobDefinitionVersion(); - return new WorkChunkCreateEvent(jobDefinitionId, jobDefinitionVersion, firstStepId, theInstanceId, 0, null); + // the first chunk of a job is always READY, no matter whether the job is gated + boolean isGatedExecution = false; + return new WorkChunkCreateEvent( + jobDefinitionId, jobDefinitionVersion, firstStepId, theInstanceId, 0, null, isGatedExecution); } @Override @@ -83,6 +92,7 @@ public class WorkChunkCreateEvent { .append(instanceId, that.instanceId) .append(sequence, that.sequence) .append(serializedData, that.serializedData) + .append(isGatedExecution, that.isGatedExecution) .isEquals(); } @@ -95,6 +105,7 @@ public class WorkChunkCreateEvent { .append(instanceId) .append(sequence) .append(serializedData) + .append(isGatedExecution) .toHashCode(); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkMetadata.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkMetadata.java new file mode 100644 index 00000000000..e06384bff75 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkMetadata.java @@ -0,0 +1,109 @@ +package ca.uhn.fhir.batch2.model; + +import ca.uhn.fhir.model.api.IModelJson; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang3.Validate; + +public class WorkChunkMetadata implements IModelJson { + + @JsonProperty("id") + private String myId; + + @JsonProperty("sequence") + // TODO MB danger - these repeat with a job or even a single step. They start at 0 for every parent chunk. Review + // after merge. + private int mySequence; + + @JsonProperty("status") + private WorkChunkStatusEnum myStatus; + + @JsonProperty("jobDefinitionId") + private String myJobDefinitionId; + + @JsonProperty("jobDefinitionVersion") + private int myJobDefinitionVersion; + + @JsonProperty("targetStepId") + private String myTargetStepId; + + @JsonProperty("instanceId") + private String myInstanceId; + + public WorkChunkStatusEnum getStatus() { + return myStatus; + } + + public WorkChunkMetadata setStatus(WorkChunkStatusEnum theStatus) { + myStatus = theStatus; + return this; + } + + public String getJobDefinitionId() { + return myJobDefinitionId; + } + + public WorkChunkMetadata setJobDefinitionId(String theJobDefinitionId) { + Validate.notBlank(theJobDefinitionId); + myJobDefinitionId = theJobDefinitionId; + return this; + } + + public int getJobDefinitionVersion() { + return myJobDefinitionVersion; + } + + public WorkChunkMetadata setJobDefinitionVersion(int theJobDefinitionVersion) { + Validate.isTrue(theJobDefinitionVersion >= 1); + myJobDefinitionVersion = theJobDefinitionVersion; + return this; + } + + public String getTargetStepId() { + return myTargetStepId; + } + + public WorkChunkMetadata setTargetStepId(String theTargetStepId) { + Validate.notBlank(theTargetStepId); + myTargetStepId = theTargetStepId; + return this; + } + + public String getInstanceId() { + return myInstanceId; + } + + public WorkChunkMetadata setInstanceId(String theInstanceId) { + myInstanceId = theInstanceId; + return this; + } + + public String getId() { + return myId; + } + + public WorkChunkMetadata setId(String theId) { + Validate.notBlank(theId); + myId = theId; + return this; + } + + public int getSequence() { + return mySequence; + } + + public void setSequence(int theSequence) { + mySequence = theSequence; + } + + public WorkChunk toWorkChunk() { + WorkChunk workChunk = new WorkChunk(); + workChunk.setId(getId()); + workChunk.setStatus(getStatus()); + workChunk.setInstanceId(getInstanceId()); + workChunk.setJobDefinitionId(getJobDefinitionId()); + workChunk.setJobDefinitionVersion(getJobDefinitionVersion()); + workChunk.setSequence(getSequence()); + workChunk.setTargetStepId(getTargetStepId()); + return workChunk; + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java index f23fdf4c153..161c3b42a83 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java @@ -31,11 +31,45 @@ import java.util.Set; * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md */ public enum WorkChunkStatusEnum { - // wipmb For 6.8 Add WAITING for gated, and READY for in db, but not yet sent to channel. + /** + * The initial state all workchunks start in for non-gated jobs. + */ + READY, + /** + * The initial state all workchunks start in for gated jobs. + */ + GATE_WAITING, + /** + * Workchunk is ready for reduction pass. + * It will not be QUEUED, but consumed inline by reduction pass. + */ + REDUCTION_READY, + /** + * The state of workchunks that have been sent to the queue; + * or of workchunks that are about to be processed in a final + * reduction step (these workchunks are never queued) + */ QUEUED, + /** + * The state of workchunks that are doing work. + */ IN_PROGRESS, + /** + * A workchunk status for workchunks that are doing long-polling work + * that will not complete in a reasonably short amount of time + */ + POLL_WAITING, + /** + * A transient state on retry when a chunk throws an error, but hasn't FAILED yet. Will move back to IN_PROGRESS on retry. + */ ERRORED, + /** + * Chunk has failed with a non-retriable error, or has run out of retry attempts. + */ FAILED, + /** + * The state of workchunks that have finished their job's step. + */ COMPLETED; private static final EnumMap> ourPriorStates; @@ -56,12 +90,25 @@ public enum WorkChunkStatusEnum { return (this != WorkChunkStatusEnum.COMPLETED); } + public boolean isAReadyState() { + return this == WorkChunkStatusEnum.READY || this == WorkChunkStatusEnum.REDUCTION_READY; + } + public Set getNextStates() { switch (this) { + case GATE_WAITING: + return EnumSet.of(READY); + case READY: + return EnumSet.of(QUEUED); + case REDUCTION_READY: + // currently no support for POLL_WAITING reduction steps + return EnumSet.of(COMPLETED, FAILED); case QUEUED: return EnumSet.of(IN_PROGRESS); case IN_PROGRESS: - return EnumSet.of(IN_PROGRESS, ERRORED, FAILED, COMPLETED); + return EnumSet.of(IN_PROGRESS, ERRORED, FAILED, COMPLETED, POLL_WAITING); + case POLL_WAITING: + return EnumSet.of(POLL_WAITING, READY); case ERRORED: return EnumSet.of(IN_PROGRESS, FAILED, COMPLETED); // terminal states diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java index 49a7bbb12de..bc335f199d1 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java @@ -48,11 +48,41 @@ * * Job and chunk processing follow state machines described {@link hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md} * Chunks have a simple {@link ca.uhn.fhir.batch2.model.WorkChunkStatusEnum state system} with states - * QUEUED, IN_PROGRESS, ERRORED, FAILED, COMPLETED. - * The initial state is QUEUED, and the final states are FAILED, and COMPLETED: + * READY, QUEUED, IN_PROGRESS, ERRORED, FAILED, COMPLETED. + * The initial state is READY, and the final states are FAILED, and COMPLETED. + * + * There are 2 primary systems in play during Batch2 Jobs. A Maintenance Job and the Batch2 Job Notification topic. + * + * The Maintenance Job + * + * This runs every minute and does the following: * *
    - *
  • Chunks are created QUEUED (NB - should be READY or WAITING) and notification is posted to the channel for non-gated steps.
    • + *
  • Moves POLL_WAITING work chunks to READY if their nextPollTime has expired.
    • + *
  • Moves READY work chunks to QUEUE and publishes it to the Batch2 Notification topic
    • + *
  • Calculates job progress (% of workchunks in complete status).
    • + *
  • If the job is finished, purges any left over work chunks.
    • + *
  • Cleans up any complete, failed, or cancelled jobs.
    • + *
  • Moves any gated jobs onto their next step.
    • + *
  • If the final step of a (gated) job is a reduction step, a reduction step execution will be triggered.
    • + *
+ * + * Processing the Messages + * + *
    + *
  • Change the work chunk from QUEUED to IN_PROGRESS
    • + *
  • Change the Job Instance status from QUEUED to IN_PROGRESS
    • + *
  • If the Job Instance is cancelled, change the status to CANCELLED and abort processing
    • + *
  • If the step creates new work chunks, each work chunk will be created in the READY state
    • + *
  • If the step succeeds, the work chunk status is changed from IN_PROGRESS to COMPLETE
    • + *
  • If the step throws a RetryChunkLaterException, the work chunk status is changed from IN_PROGRESS to POLL_WAITING and a nextPollTime value set.
    • + *
  • If the step fails, the work chunk status is changed from IN_PROGRESS to either ERRORED or FAILED depending on the severity of the error
    • + *
+ * + * The job lifecycle + * + *
    + *
  • Chunks are created READY (NB - should be READY or WAITING) and notification is posted to the channel for non-gated steps.
    • *
  • * Workers receive a notification and advance QUEUED->IN_PROGRESS. * {@link ca.uhn.fhir.batch2.api.IWorkChunkPersistence#onWorkChunkDequeue(String)} @@ -60,6 +90,7 @@ *
  • * On normal execution, the chunk advances IN_PROGRESS->COMPLETED {@link ca.uhn.fhir.batch2.api.IWorkChunkPersistence#onWorkChunkCompletion}
    • *
  • On a retryiable error, IN_PROGRESS->ERROR with an error message and the chunk is put back on the queue. {@link ca.uhn.fhir.batch2.api.IWorkChunkPersistence#onWorkChunkError}
    • + *
  • On a RetryChunkLaterException, IN_PROGRESS->POLL_WAITING with a nextPollTime set. The chunk is *not* put back on the queue, but is left for the maintenance job to manage.
    • *
  • On a hard failure, or too many errors, IN_PROGRESS->FAILED with the error message. {@link ca.uhn.fhir.batch2.api.IWorkChunkPersistence#onWorkChunkFailed}
    • *
* diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java index 790ed970c1a..c38c7a5cc09 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java @@ -73,7 +73,10 @@ public class InstanceProgress { statusToCountMap.put(theChunk.getStatus(), statusToCountMap.getOrDefault(theChunk.getStatus(), 0) + 1); switch (theChunk.getStatus()) { + case GATE_WAITING: + case READY: case QUEUED: + case POLL_WAITING: case IN_PROGRESS: myIncompleteChunkCount++; break; diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java index 348fd30e540..fcaecb1ce2f 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java @@ -57,6 +57,7 @@ public class JobInstanceProgressCalculator { StopWatch stopWatch = new StopWatch(); ourLog.trace("calculating progress: {}", theInstanceId); + // calculate progress based on number of work chunks in COMPLETE state InstanceProgress instanceProgress = calculateInstanceProgress(theInstanceId); myJobPersistence.updateInstance(theInstanceId, currentInstance -> { @@ -97,6 +98,7 @@ public class JobInstanceProgressCalculator { while (workChunkIterator.hasNext()) { WorkChunk next = workChunkIterator.next(); + // global stats myProgressAccumulator.addChunk(next); // instance stats diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java index ebd682f2386..959cb3e0569 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelReceiver; import ca.uhn.fhir.jpa.subscription.channel.impl.LinkedBlockingChannel; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterEach; @@ -54,6 +55,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -73,7 +75,7 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { private JobDefinitionRegistry myJobDefinitionRegistry; @Mock private IJobMaintenanceService myJobMaintenanceService; - private IHapiTransactionService myTransactionService = new NonTransactionalHapiTransactionService(); + private final IHapiTransactionService myTransactionService = new NonTransactionalHapiTransactionService(); @Captor private ArgumentCaptor> myStep1ExecutionDetailsCaptor; @Captor @@ -144,7 +146,6 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { assertEquals(PASSWORD_VALUE, params.getPassword()); verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 50, 0)); - verify(myBatchJobSender, times(2)).sendWorkChannelMessage(any()); } private void setupMocks(JobDefinition theJobDefinition, WorkChunk theWorkChunk) { @@ -183,7 +184,7 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { .thenReturn(Arrays.asList(existingInProgInstance)); // test - Batch2JobStartResponse startResponse = mySvc.startInstance(startRequest); + Batch2JobStartResponse startResponse = mySvc.startInstance(new SystemRequestDetails(), startRequest); // verify assertEquals(inProgressInstanceId, startResponse.getInstanceId()); // make sure it's the completed one @@ -467,7 +468,7 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { JobInstanceStartRequest startRequest = new JobInstanceStartRequest(); startRequest.setJobDefinitionId(JOB_DEFINITION_ID); startRequest.setParameters(new TestJobParameters().setParam1(PARAM_1_VALUE).setParam2(PARAM_2_VALUE).setPassword(PASSWORD_VALUE)); - mySvc.startInstance(startRequest); + mySvc.startInstance(new SystemRequestDetails(), startRequest); // Verify @@ -476,12 +477,7 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { assertSame(jobDefinition, myJobDefinitionCaptor.getValue()); assertEquals(startRequest.getParameters(), myParametersJsonCaptor.getValue()); - verify(myBatchJobSender, times(1)).sendWorkChannelMessage(myJobWorkNotificationCaptor.capture()); - assertEquals(CHUNK_ID, myJobWorkNotificationCaptor.getAllValues().get(0).getChunkId()); - assertEquals(JOB_DEFINITION_ID, myJobWorkNotificationCaptor.getAllValues().get(0).getJobDefinitionId()); - assertEquals(1, myJobWorkNotificationCaptor.getAllValues().get(0).getJobDefinitionVersion()); - assertEquals(STEP_1, myJobWorkNotificationCaptor.getAllValues().get(0).getTargetStepId()); - + verify(myBatchJobSender, never()).sendWorkChannelMessage(any()); verifyNoMoreInteractions(myJobInstancePersister); verifyNoMoreInteractions(myStep1Worker); verifyNoMoreInteractions(myStep2Worker); diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java index 5b2f89dae96..fd28ccae545 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java @@ -29,12 +29,18 @@ import org.mockito.junit.jupiter.MockitoExtension; import jakarta.annotation.Nonnull; import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -99,6 +105,11 @@ class JobDataSinkTest { // execute // Let's test our first step worker by calling run on it: when(myJobPersistence.onWorkChunkCreate(myBatchWorkChunkCaptor.capture())).thenReturn(CHUNK_ID); + doAnswer(args -> { + Consumer consumer = args.getArgument(1); + consumer.accept(1); + return 1; + }).when(myJobPersistence).enqueueWorkChunkForProcessing(anyString(), any()); JobInstance instance = JobInstance.fromInstanceId(JOB_INSTANCE_ID); StepExecutionDetails details = new StepExecutionDetails<>(new TestJobParameters().setParam1("" + PID_COUNT), null, instance, CHUNK_ID); JobWorkCursor cursor = new JobWorkCursor<>(job, true, firstStep, lastStep); @@ -112,7 +123,6 @@ class JobDataSinkTest { // theDataSink.accept(output) called by firstStepWorker above calls two services. Let's validate them both. verify(myBatchJobSender).sendWorkChannelMessage(myJobWorkNotificationCaptor.capture()); - JobWorkNotification notification = myJobWorkNotificationCaptor.getValue(); assertEquals(JOB_DEF_ID, notification.getJobDefinitionId()); assertEquals(JOB_INSTANCE_ID, notification.getInstanceId()); diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java index 22aa7ba9c30..e080586547c 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java @@ -20,6 +20,8 @@ import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -62,10 +64,6 @@ public class ReductionStepExecutorServiceImplTest { private IJobPersistence myJobPersistence; @Mock private IReductionStepWorker myReductionStepWorker; - // @Mock -// private JobDefinitionStep myPreviousStep; -// @Mock -// private JobDefinitionStep myCurrentStep; private ReductionStepExecutorServiceImpl mySvc; private final JobDefinitionRegistry myJobDefinitionRegistry = new JobDefinitionRegistry(); @@ -74,13 +72,19 @@ public class ReductionStepExecutorServiceImplTest { mySvc = new ReductionStepExecutorServiceImpl(myJobPersistence, myTransactionService, myJobDefinitionRegistry); } - @Test + // QUEUED, IN_PROGRESS are supported because of backwards compatibility + // these statuses will stop being supported after 7.6 + @SuppressWarnings({"unchecked", "rawtypes"}) + @ParameterizedTest + @EnumSource(value = WorkChunkStatusEnum.class, names = { "REDUCTION_READY", "QUEUED", "IN_PROGRESS" }) public void doExecution_reductionWithChunkFailed_marksAllFutureChunksAsFailedButPreviousAsSuccess() { // setup List chunkIds = Arrays.asList("chunk1", "chunk2"); List chunks = new ArrayList<>(); for (String id : chunkIds) { - chunks.add(createWorkChunk(id)); + WorkChunk chunk = createWorkChunk(id); + chunk.setStatus(WorkChunkStatusEnum.REDUCTION_READY); + chunks.add(chunk); } JobInstance jobInstance = getTestJobInstance(); jobInstance.setStatus(StatusEnum.IN_PROGRESS); @@ -125,20 +129,21 @@ public class ReductionStepExecutorServiceImplTest { assertEquals(WorkChunkStatusEnum.FAILED, statuses.get(1)); } - + @SuppressWarnings({"unchecked", "rawtypes"}) @Test public void doExecution_reductionStepWithValidInput_executesAsExpected() { // setup List chunkIds = Arrays.asList("chunk1", "chunk2"); List chunks = new ArrayList<>(); for (String id : chunkIds) { - chunks.add(createWorkChunk(id)); + WorkChunk chunk = createWorkChunk(id); + chunk.setStatus(WorkChunkStatusEnum.REDUCTION_READY); + chunks.add(chunk); } JobInstance jobInstance = getTestJobInstance(); jobInstance.setStatus(StatusEnum.IN_PROGRESS); JobWorkCursor workCursor = mock(JobWorkCursor.class); - // when when(workCursor.getCurrentStep()).thenReturn((JobDefinitionStep) createJobDefinition().getSteps().get(1)); when(workCursor.getJobDefinition()).thenReturn(createJobDefinition()); @@ -176,14 +181,17 @@ public class ReductionStepExecutorServiceImplTest { } - + @SuppressWarnings({"unchecked", "rawtypes"}) @Test public void doExecution_reductionStepWithErrors_returnsFalseAndMarksPreviousChunksFailed() { // setup List chunkIds = Arrays.asList("chunk1", "chunk2"); List chunks = new ArrayList<>(); for (String id : chunkIds) { - chunks.add(createWorkChunk(id)); + WorkChunk chunk = createWorkChunk(id); + // reduction steps are done with REDUCTION_READY workchunks + chunk.setStatus(WorkChunkStatusEnum.REDUCTION_READY); + chunks.add(chunk); } JobInstance jobInstance = getTestJobInstance(); jobInstance.setStatus(StatusEnum.IN_PROGRESS); diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java index 10f1a007e5c..e107cf106f3 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java @@ -488,7 +488,7 @@ public class WorkChunkProcessorTest { WorkChunk chunk = new WorkChunk(); chunk.setInstanceId(INSTANCE_ID); chunk.setId(theId); - chunk.setStatus(WorkChunkStatusEnum.QUEUED); + chunk.setStatus(WorkChunkStatusEnum.READY); chunk.setData(JsonUtil.serialize( new StepInputData() )); diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java index ba11ac13560..f901ee885c2 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java @@ -15,11 +15,11 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkMetadata; import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; import ca.uhn.fhir.jpa.model.sched.ISchedulerService; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelProducer; -import ca.uhn.fhir.util.Logs; import ca.uhn.test.util.LogbackCaptureTestExtension; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; @@ -36,20 +36,29 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; import org.springframework.messaging.Message; +import org.springframework.transaction.PlatformTransactionManager; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.function.Consumer; import java.util.stream.Collectors; import static ca.uhn.fhir.batch2.coordinator.JobCoordinatorImplTest.createWorkChunkStep1; +import static ca.uhn.fhir.batch2.coordinator.JobCoordinatorImplTest.createWorkChunkStep2; +import static ca.uhn.fhir.batch2.coordinator.JobCoordinatorImplTest.createWorkChunkStep3; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -59,7 +68,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -85,6 +97,8 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { private JobDefinitionRegistry myJobDefinitionRegistry; @Mock private IChannelProducer myWorkChannelProducer; + @Mock + private PlatformTransactionManager myTransactionService; @Captor private ArgumentCaptor> myMessageCaptor; @Captor @@ -102,7 +116,8 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { myJobDefinitionRegistry, batchJobSender, myJobExecutorSvc, - myReductionStepExecutorService); + myReductionStepExecutorService + ); myStorageSettings.setJobFastTrackingEnabled(true); } @@ -114,11 +129,14 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { ); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); + Page page = getPageOfData(chunks); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); JobInstance instance = createInstance(); when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(List.of(instance)); when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(INSTANCE_ID), any())) + .thenReturn(page); mySvc.runMaintenancePass(); @@ -153,7 +171,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); JobInstance instance = createInstance(); @@ -161,6 +179,8 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(instance.getInstanceId()), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenReturn(Page.empty()); stubUpdateInstanceCallback(instance); mySvc.runMaintenancePass(); @@ -175,6 +195,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { assertNull(instance.getEndTime()); assertEquals("00:10:00", instance.getEstimatedTimeRemaining()); + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(instance.getInstanceId())); verifyNoMoreInteractions(myJobPersistence); } @@ -194,7 +215,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setErrorCount(2), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setErrorCount(2), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); JobInstance instance = createInstance(); @@ -203,6 +224,8 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(instance.getInstanceId()), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenReturn(Page.empty()); stubUpdateInstanceCallback(instance); // Execute @@ -217,30 +240,40 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { assertEquals(50, instance.getCombinedRecordsProcessed()); assertEquals(0.08333333333333333, instance.getCombinedRecordsProcessedPerSecond()); + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(instance.getInstanceId())); verifyNoMoreInteractions(myJobPersistence); } @Test public void testInProgress_GatedExecution_FirstStepComplete() { // Setup + List completedChunks = List.of(createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setId(CHUNK_ID)); + List chunks = Arrays.asList( - JobCoordinatorImplTest.createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setId(CHUNK_ID + "abc"), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.QUEUED).setId(CHUNK_ID), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.GATE_WAITING).setId(CHUNK_ID), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.QUEUED).setId(CHUNK_ID_2) ); - when (myJobPersistence.canAdvanceInstanceToNextStep(any(), any())).thenReturn(true); myJobDefinitionRegistry.addJobDefinition(createJobDefinition(JobDefinition.Builder::gatedExecution)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); - - when(myJobPersistence.fetchAllChunkIdsForStepWithStatus(eq(INSTANCE_ID), eq(STEP_2), eq(WorkChunkStatusEnum.QUEUED))) - .thenReturn(chunks.stream().filter(c->c.getTargetStepId().equals(STEP_2)).map(WorkChunk::getId).collect(Collectors.toList())); + when(myJobPersistence.getDistinctWorkChunkStatesForJobAndStep(anyString(), anyString())) + .thenReturn(completedChunks.stream().map(WorkChunk::getStatus).collect(Collectors.toSet())); JobInstance instance1 = createInstance(); instance1.setCurrentGatedStepId(STEP_1); when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance1)); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), anyString(), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenAnswer((args) -> { + // new page every time (called more than once) + return getPageOfData(new ArrayList<>(chunks)); + }); + doAnswer(a -> { + Consumer callback = a.getArgument(1); + callback.accept(1); + return null; + }).when(myJobPersistence).enqueueWorkChunkForProcessing(anyString(), any()); stubUpdateInstanceCallback(instance1); // Execute @@ -248,7 +281,9 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { // Verify verify(myWorkChannelProducer, times(2)).send(myMessageCaptor.capture()); - verify(myJobPersistence, times(2)).updateInstance(eq(INSTANCE_ID), any()); + verify(myJobPersistence, times(1)).updateInstance(eq(INSTANCE_ID), any()); + verify(myJobPersistence, times(1)).advanceJobStepAndUpdateChunkStatus(eq(INSTANCE_ID), eq(STEP_2), eq(false)); + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(INSTANCE_ID)); verifyNoMoreInteractions(myJobPersistence); JobWorkNotification payload0 = myMessageCaptor.getAllValues().get(0).getPayload(); assertEquals(STEP_2, payload0.getTargetStepId()); @@ -266,10 +301,13 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { instance.setEndTime(parseTime("2001-01-01T12:12:12Z")); when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(instance.getInstanceId()), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenReturn(Page.empty()); mySvc.runMaintenancePass(); verify(myJobPersistence, times(1)).deleteInstanceAndChunks(eq(INSTANCE_ID)); + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(instance.getInstanceId())); verifyNoMoreInteractions(myJobPersistence); } @@ -281,7 +319,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:01-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25),JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition(t -> t.completionHandler(myCompletionHandler))); @@ -289,6 +327,8 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())).thenAnswer(t->chunks.iterator()); when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(INSTANCE_ID), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenReturn(Page.empty()); stubUpdateInstanceCallback(instance); // Execute @@ -307,7 +347,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { verify(myJobPersistence, times(1)).deleteChunksAndMarkInstanceAsChunksPurged(eq(INSTANCE_ID)); verify(myCompletionHandler, times(1)).jobComplete(myJobCompletionCaptor.capture()); - + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(instance.getInstanceId())); verifyNoMoreInteractions(myJobPersistence); assertEquals(INSTANCE_ID, myJobCompletionCaptor.getValue().getInstance().getInstanceId()); @@ -322,7 +362,7 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.FAILED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25).setErrorMessage("This is an error message"), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); @@ -331,11 +371,12 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())) .thenAnswer(t->chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(instance.getInstanceId()), eq(Set.of(WorkChunkStatusEnum.READY)))) + .thenReturn(Page.empty()); stubUpdateInstanceCallback(instance); mySvc.runMaintenancePass(); - assertEquals(0.8333333333333334, instance.getProgress()); assertEquals(StatusEnum.FAILED, instance.getStatus()); assertEquals("This is an error message", instance.getErrorMessage()); @@ -346,10 +387,134 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { // twice - once to move to FAILED, and once to purge the chunks verify(myJobPersistence, times(1)).updateInstance(eq(INSTANCE_ID), any()); verify(myJobPersistence, times(1)).deleteChunksAndMarkInstanceAsChunksPurged(eq(INSTANCE_ID)); - + verify(myJobPersistence).updatePollWaitingChunksForJobIfReady(eq(instance.getInstanceId())); verifyNoMoreInteractions(myJobPersistence); } + private void runEnqueueReadyChunksTest(List theChunks, JobDefinition theJobDefinition) { + myJobDefinitionRegistry.addJobDefinition(theJobDefinition); + JobInstance instance = createInstance(); + // we'll set the instance to the first step id + theChunks.stream().findFirst().ifPresent(c -> { + instance.setCurrentGatedStepId(c.getTargetStepId()); + }); + instance.setJobDefinitionId(theJobDefinition.getJobDefinitionId()); + + // mocks + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); + when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())) + .thenAnswer(t -> theChunks.stream().map(c -> c.setStatus(WorkChunkStatusEnum.READY)).toList().iterator()); + + // test + mySvc.runMaintenancePass(); + } + + @Test + public void testMaintenancePass_withREADYWorkChunksForReductionSteps_notQueuedButProcessed() { + // setup + List chunks = List.of( + createWorkChunkStep3().setStatus(WorkChunkStatusEnum.READY), + createWorkChunkStep3().setStatus(WorkChunkStatusEnum.READY) + ); + List previousChunks = List.of( + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED), + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED) + ); + + String lastStepId = chunks.get(0).getTargetStepId(); + + // when + when(myJobPersistence.getDistinctWorkChunkStatesForJobAndStep(eq(INSTANCE_ID), eq(lastStepId))) + .thenReturn(chunks.stream().map(WorkChunk::getStatus).collect(Collectors.toSet())); + + // test + runEnqueueReadyChunksTest(chunks, createJobDefinitionWithReduction()); + + // verify never updated (should remain in ready state) + verify(myJobPersistence, never()).fetchAllWorkChunkMetadataForJobInStates(any(), anyString(), any()); + verify(myJobPersistence, never()).enqueueWorkChunkForProcessing(anyString(), any()); + verify(myWorkChannelProducer, never()).send(any()); + verify(myReductionStepExecutorService) + .triggerReductionStep(anyString(), any()); + } + + @Test + public void testMaintenancePass_withREADYworkChunksForNonReductionStep_movedToQUEUEDandPublished() { + // setup + List chunks = List.of( + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.READY), + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.READY) + ); + + // when + doAnswer(args -> { + Consumer consumer = args.getArgument(1); + consumer.accept(1); + return 1; + }).when(myJobPersistence).enqueueWorkChunkForProcessing(anyString(), any()); + + Page page = getPageOfData(chunks); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(INSTANCE_ID), any())).thenReturn(page); + + // test + runEnqueueReadyChunksTest(chunks, createJobDefinition()); + + // verify + verify(myJobPersistence, times(2)).enqueueWorkChunkForProcessing(anyString(), any()); + verify(myWorkChannelProducer, times(2)).send(myMessageCaptor.capture()); + List> sentMessages = myMessageCaptor.getAllValues(); + for (Message msg : sentMessages) { + JobWorkNotification payload = msg.getPayload(); + assertEquals(STEP_2, payload.getTargetStepId()); + assertEquals(CHUNK_ID, payload.getChunkId()); + } + } + + @Test + public void testMaintenancePass_whenUpdateFails_skipsWorkChunkAndLogs() { + // setup + List chunks = List.of( + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.READY), + createWorkChunkStep2().setStatus(WorkChunkStatusEnum.READY) + ); + JobInstance instance = createInstance(); + instance.setCurrentGatedStepId(STEP_2); + + myLogCapture.setUp(Level.ERROR); + + // when + doAnswer(args -> { + Consumer consumer = args.getArgument(1); + consumer.accept(0); // nothing processed + return 1; + }).when(myJobPersistence).enqueueWorkChunkForProcessing(anyString(), any()); + doAnswer(args -> { + IJobPersistence.JobInstanceUpdateCallback callback = args.getArgument(1); + + callback.doUpdate(instance); + return true; + }).when(myJobPersistence).updateInstance(any(), any()); + when(myJobPersistence.getDistinctWorkChunkStatesForJobAndStep(eq(instance.getInstanceId()), eq(STEP_2))) + .thenReturn(chunks.stream().map(WorkChunkMetadata::getStatus).collect(Collectors.toSet())); + Page page = getPageOfData(chunks); + when(myJobPersistence.fetchAllWorkChunkMetadataForJobInStates(any(Pageable.class), eq(INSTANCE_ID), any())).thenReturn(page); + + + // test + runEnqueueReadyChunksTest(chunks, createJobDefinitionWithReduction()); + + // verify + verify(myJobPersistence, times(2)).enqueueWorkChunkForProcessing(anyString(), any()); + verify(myWorkChannelProducer, never()).send(any()); + + List events = myLogCapture.getLogEvents(); + assertEquals(2, events.size()); + for (ILoggingEvent evt : events) { + assertTrue(evt.getMessage().contains("skipping work chunk")); + } + } + @Test void triggerMaintenancePass_noneInProgress_runsMaintenance() { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Collections.emptyList()); @@ -408,9 +573,11 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { assertTrue(result2.get()); } - private static Date parseTime(String theDate) { return new DateTimeType(theDate).getValue(); } + private Page getPageOfData(List theChunks) { + return new PageImpl<>(theChunks.stream().map(c -> (WorkChunkMetadata)c).collect(Collectors.toList())); + } } diff --git a/hapi-fhir-storage-cr/pom.xml b/hapi-fhir-storage-cr/pom.xml index bcdfc3b4916..a5487ccedb9 100644 --- a/hapi-fhir-storage-cr/pom.xml +++ b/hapi-fhir-storage-cr/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-mdm/pom.xml b/hapi-fhir-storage-mdm/pom.xml index 23727b3009c..669d31c487a 100644 --- a/hapi-fhir-storage-mdm/pom.xml +++ b/hapi-fhir-storage-mdm/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-test-utilities/pom.xml b/hapi-fhir-storage-test-utilities/pom.xml index f6a72dbc6ab..1027500637c 100644 --- a/hapi-fhir-storage-test-utilities/pom.xml +++ b/hapi-fhir-storage-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage/pom.xml b/hapi-fhir-storage/pom.xml index 1ff49ea3ab7..6cd1fd9b443 100644 --- a/hapi-fhir-storage/pom.xml +++ b/hapi-fhir-storage/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu2.1/pom.xml b/hapi-fhir-structures-dstu2.1/pom.xml index 43a2ba0fab3..059bd1b0451 100644 --- a/hapi-fhir-structures-dstu2.1/pom.xml +++ b/hapi-fhir-structures-dstu2.1/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu2/pom.xml b/hapi-fhir-structures-dstu2/pom.xml index 2138b6f052c..4d0b3bdd812 100644 --- a/hapi-fhir-structures-dstu2/pom.xml +++ b/hapi-fhir-structures-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index c94c9e07e17..586b61d07ac 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-hl7org-dstu2/pom.xml b/hapi-fhir-structures-hl7org-dstu2/pom.xml index c380b9e13a3..2442012db29 100644 --- a/hapi-fhir-structures-hl7org-dstu2/pom.xml +++ b/hapi-fhir-structures-hl7org-dstu2/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index ba5db156616..d9472c2b050 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/ConsentInterceptorTest.java b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/ConsentInterceptorTest.java index 237e72a218c..a1acb0fcc9e 100644 --- a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/ConsentInterceptorTest.java +++ b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/interceptor/ConsentInterceptorTest.java @@ -262,6 +262,7 @@ public class ConsentInterceptorTest { verify(myConsentSvc, timeout(2000).times(0)).willSeeResource(any(), any(), any()); verify(myConsentSvc, timeout(2000).times(0)).startOperation(any(), any()); verify(myConsentSvc, timeout(2000).times(2)).completeOperationSuccess(any(), any()); + verifyNoMoreInteractions(myConsentSvc); } @@ -887,7 +888,8 @@ public class ConsentInterceptorTest { assertEquals(2, response.getTotal()); } - @Nested class CacheUsage { + @Nested + class CacheUsage { @Mock ICachedSearchDetails myCachedSearchDetails; ServletRequestDetails myRequestDetails = new ServletRequestDetails(); diff --git a/hapi-fhir-structures-r4b/pom.xml b/hapi-fhir-structures-r4b/pom.xml index 452563edf9d..7b297c89f32 100644 --- a/hapi-fhir-structures-r4b/pom.xml +++ b/hapi-fhir-structures-r4b/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r5/pom.xml b/hapi-fhir-structures-r5/pom.xml index 270596ab0cd..d833b3419cf 100644 --- a/hapi-fhir-structures-r5/pom.xml +++ b/hapi-fhir-structures-r5/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-test-utilities/pom.xml b/hapi-fhir-test-utilities/pom.xml index dc5ad4d0fc8..f93fcb3edfe 100644 --- a/hapi-fhir-test-utilities/pom.xml +++ b/hapi-fhir-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/LogbackLevelOverrideExtension.java b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/LogbackLevelOverrideExtension.java index 3188ffa498b..907f378b86e 100644 --- a/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/LogbackLevelOverrideExtension.java +++ b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/LogbackLevelOverrideExtension.java @@ -38,10 +38,14 @@ public class LogbackLevelOverrideExtension implements AfterEachCallback { public void setLogLevel(Class theClass, Level theLevel) { String name = theClass.getName(); - Logger logger = getClassicLogger(name); - if (!mySavedLevels.containsKey(name)) { + setLogLevel(name, theLevel); + } + + public void setLogLevel(String theName, Level theLevel) { + Logger logger = getClassicLogger(theName); + if (!mySavedLevels.containsKey(theName)) { // level can be null - mySavedLevels.put(name, logger.getLevel()); + mySavedLevels.put(theName, logger.getLevel()); } logger.setLevel(theLevel); } diff --git a/hapi-fhir-testpage-overlay/pom.xml b/hapi-fhir-testpage-overlay/pom.xml index 162dc758563..d8607ca6c3b 100644 --- a/hapi-fhir-testpage-overlay/pom.xml +++ b/hapi-fhir-testpage-overlay/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-validation-resources-dstu2.1/pom.xml b/hapi-fhir-validation-resources-dstu2.1/pom.xml index 4458b64cc8b..828c0364fc5 100644 --- a/hapi-fhir-validation-resources-dstu2.1/pom.xml +++ b/hapi-fhir-validation-resources-dstu2.1/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu2/pom.xml b/hapi-fhir-validation-resources-dstu2/pom.xml index 3d923001ddb..17eeee20920 100644 --- a/hapi-fhir-validation-resources-dstu2/pom.xml +++ b/hapi-fhir-validation-resources-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu3/pom.xml b/hapi-fhir-validation-resources-dstu3/pom.xml index 1dcfb04cbbf..69a3445b187 100644 --- a/hapi-fhir-validation-resources-dstu3/pom.xml +++ b/hapi-fhir-validation-resources-dstu3/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r4/pom.xml b/hapi-fhir-validation-resources-r4/pom.xml index f2e3aa6f5c0..157973beb1c 100644 --- a/hapi-fhir-validation-resources-r4/pom.xml +++ b/hapi-fhir-validation-resources-r4/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r4b/pom.xml b/hapi-fhir-validation-resources-r4b/pom.xml index 1af75ea43e2..1d364151cae 100644 --- a/hapi-fhir-validation-resources-r4b/pom.xml +++ b/hapi-fhir-validation-resources-r4b/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r5/pom.xml b/hapi-fhir-validation-resources-r5/pom.xml index 989a90079bf..f7129e424c0 100644 --- a/hapi-fhir-validation-resources-r5/pom.xml +++ b/hapi-fhir-validation-resources-r5/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation/pom.xml b/hapi-fhir-validation/pom.xml index 347c73a1527..6b0d189e778 100644 --- a/hapi-fhir-validation/pom.xml +++ b/hapi-fhir-validation/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-tinder-plugin/pom.xml b/hapi-tinder-plugin/pom.xml index 34cfa205251..133e5be72dd 100644 --- a/hapi-tinder-plugin/pom.xml +++ b/hapi-tinder-plugin/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/hapi-tinder-test/pom.xml b/hapi-tinder-test/pom.xml index aa0dda090ec..a580dcc60b2 100644 --- a/hapi-tinder-test/pom.xml +++ b/hapi-tinder-test/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 9abd2416429..b2bcabda6e9 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ ca.uhn.hapi.fhir hapi-fhir pom - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT HAPI-FHIR An open-source implementation of the FHIR specification in Java. diff --git a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml index 54410998f56..c7e71349f01 100644 --- a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml +++ b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-client/pom.xml b/tests/hapi-fhir-base-test-mindeps-client/pom.xml index b0de40fff10..f0e456b6846 100644 --- a/tests/hapi-fhir-base-test-mindeps-client/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-server/pom.xml b/tests/hapi-fhir-base-test-mindeps-server/pom.xml index 14cdca64555..1e5ce59b61c 100644 --- a/tests/hapi-fhir-base-test-mindeps-server/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 7.3.0-SNAPSHOT + 7.3.1-SNAPSHOT ../../pom.xml