From c8312662d71638384572232c925535d864425408 Mon Sep 17 00:00:00 2001 From: Luke deGruchy Date: Thu, 20 Oct 2022 14:27:01 -0400 Subject: [PATCH] Set batch export job results count to number of resources processed instead of 0. (#4172) * Add count to batch export admin page results and new add new unit test. * Fix failing unit test in WriteBinaryStepTest. * Add changelog. * Small changelog fix. --- .../4173-bulk-export-resource-count-zero.yaml | 4 ++ .../fhir/jpa/bulk/BulkExportUseCaseTest.java | 46 ++++++++++++++++++- .../batch2/jobs/export/WriteBinaryStep.java | 5 +- .../jobs/export/WriteBinaryStepTest.java | 2 +- 4 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4173-bulk-export-resource-count-zero.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4173-bulk-export-resource-count-zero.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4173-bulk-export-resource-count-zero.yaml new file mode 100644 index 00000000000..a0f128c5358 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4173-bulk-export-resource-count-zero.yaml @@ -0,0 +1,4 @@ +--- +type: fix +issue: 4173 +title: "When triggering a batch export, the number of resources processed is returned as zero. This is now fixed to return the total number of resources processed across a single or multiple bundles." diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java index 503f2831d27..f9c969106e8 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java @@ -1,5 +1,7 @@ package ca.uhn.fhir.jpa.bulk; +import ca.uhn.fhir.batch2.api.IJobPersistence; +import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; import ca.uhn.fhir.jpa.api.svc.IBatch2JobRunner; @@ -44,6 +46,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.is; @@ -56,6 +60,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { private static final Logger ourLog = LoggerFactory.getLogger(BulkExportUseCaseTest.class); @@ -63,6 +68,9 @@ public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { @Autowired private IBatch2JobRunner myJobRunner; + @Autowired + private IJobPersistence myJobPersistence; + @Nested public class SpecConformanceTests { @Test @@ -171,6 +179,40 @@ public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { } } + @Test + public void testResourceCountIsCorrect() { + int patientCount = 5; + for (int i = 0; i < patientCount; i++) { + Patient patient = new Patient(); + patient.setId("pat-" + i); + myPatientDao.update(patient); + } + + BulkDataExportOptions options = new BulkDataExportOptions(); + options.setResourceTypes(Collections.singleton("Patient")); + options.setFilters(Collections.emptySet()); + options.setExportStyle(BulkDataExportOptions.ExportStyle.SYSTEM); + options.setOutputFormat(Constants.CT_FHIR_NDJSON); + + Batch2JobStartResponse startResponse = myJobRunner.startNewJob(BulkExportUtils.createBulkExportJobParametersFromExportOptions(options)); + + assertNotNull(startResponse); + + final String jobId = startResponse.getJobId(); + + // Run a scheduled pass to build the export + myBatch2JobHelper.awaitJobCompletion(startResponse.getJobId()); + + final Optional optJobInstance = myJobPersistence.fetchInstance(jobId); + + assertNotNull(optJobInstance); + assertTrue(optJobInstance.isPresent()); + + final JobInstance jobInstance = optJobInstance.get(); + + assertEquals(patientCount, jobInstance.getCombinedRecordsProcessed()); + } + private void logContentTypeAndResponse(Header[] headers, String response) { ourLog.info("**************************"); ourLog.info("Content-Type is: {}", headers[0]); @@ -705,11 +747,11 @@ public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { return startBulkExportJobAndAwaitCompletion(BulkDataExportOptions.ExportStyle.GROUP, theResourceTypes, theFilters, theGroupId); } - BulkExportJobResults startSystemBulkExportJobAndAwaitCompletion(HashSet theResourceTypes, HashSet theFilters) { + BulkExportJobResults startSystemBulkExportJobAndAwaitCompletion(Set theResourceTypes, Set theFilters) { return startBulkExportJobAndAwaitCompletion(BulkDataExportOptions.ExportStyle.SYSTEM, theResourceTypes, theFilters, null); } - BulkExportJobResults startBulkExportJobAndAwaitCompletion(BulkDataExportOptions.ExportStyle theExportStyle, HashSet theResourceTypes, HashSet theFilters, String theGroupOrPatientId) { + BulkExportJobResults startBulkExportJobAndAwaitCompletion(BulkDataExportOptions.ExportStyle theExportStyle, Set theResourceTypes, Set theFilters, String theGroupOrPatientId) { BulkDataExportOptions options = new BulkDataExportOptions(); options.setResourceTypes(theResourceTypes); options.setFilters(theFilters); diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java index 9c7f05b1589..eed3681c5c0 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java @@ -64,9 +64,10 @@ public class WriteBinaryStep implements IJobStepWorker theDataSink) throws JobExecutionFailedException { BulkExportExpandedResources expandedResources = theStepExecutionDetails.getData(); + final int numResourcesProcessed = expandedResources.getStringifiedResources().size(); ourLog.info("Write binary step of Job Export"); - ourLog.info("Writing {} resources to binary file", expandedResources.getStringifiedResources().size()); + ourLog.info("Writing {} resources to binary file", numResourcesProcessed); @SuppressWarnings("unchecked") IFhirResourceDao binaryDao = myDaoRegistry.getResourceDao("Binary"); @@ -113,7 +114,7 @@ public class WriteBinaryStep implements IJobStepWorker binaryCaptor = ArgumentCaptor.forClass(IBaseBinary.class); verify(binaryDao)