From e4a2285f30c05bd2f89a7dbf740517bc5706f6bc Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 28 Sep 2022 11:16:15 -0400 Subject: [PATCH] Make sure to not mix resource types in bulk export (#4093) * Make sure to not mix resource types in bulk export * Avoid dupes * Add test --- ...prevent-mixed-restypes-in-bulk-export.yaml | 5 + .../uhn/fhir/jpa/bulk/BulkDataExportTest.java | 147 ++++++++++++------ .../jobs/export/ExpandResourcesStep.java | 39 +++-- .../models/BulkExportExpandedResources.java | 1 + .../jobs/export/ExpandResourcesStepTest.java | 7 - .../jobs/export/WriteBinaryStepTest.java | 4 +- 6 files changed, 130 insertions(+), 73 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4093-prevent-mixed-restypes-in-bulk-export.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4093-prevent-mixed-restypes-in-bulk-export.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4093-prevent-mixed-restypes-in-bulk-export.yaml new file mode 100644 index 00000000000..dcae9e71040 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4093-prevent-mixed-restypes-in-bulk-export.yaml @@ -0,0 +1,5 @@ +--- +type: fix +issue: 4093 +title: "A previous fix resulted in Bulk Export files containing mixed resource types, which is + not allowed in the bulk data access IG. This has been corrected." diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java index 3e321236a21..22ea23c5c22 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java @@ -10,7 +10,10 @@ import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; import ca.uhn.fhir.util.JsonUtil; import com.google.common.collect.Sets; +import org.apache.commons.io.LineIterator; import org.hamcrest.Matchers; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Binary; import org.hl7.fhir.r4.model.Device; import org.hl7.fhir.r4.model.Encounter; @@ -30,16 +33,22 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import java.io.IOException; +import java.io.StringReader; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; public class BulkDataExportTest extends BaseResourceProviderR4Test { private static final Logger ourLog = LoggerFactory.getLogger(BulkDataExportTest.class); @@ -84,11 +93,11 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setFilters(Sets.newHashSet("Patient?gender=female")); options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, Collections.singletonList("\"PF\""), Collections.singletonList("\"PM\"")); + verifyBulkExportResults(options, Collections.singletonList("Patient/PF"), Collections.singletonList("Patient/PM")); } @Test - public void testGroupBulkExportNotInGroup_DoeNotShowUp() { + public void testGroupBulkExportNotInGroup_DoesNotShowUp() { // Create some resources Patient patient = new Patient(); patient.setId("PING1"); @@ -122,7 +131,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setFilters(new HashSet<>()); options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, List.of("\"PING1\"", "\"PING2\""), Collections.singletonList("\"PNING3\"")); + verifyBulkExportResults(options, List.of("Patient/PING1", "Patient/PING2"), Collections.singletonList("Patient/PNING3")); } @Test @@ -149,12 +158,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setOutputFormat(Constants.CT_FHIR_NDJSON); myCaptureQueriesListener.clear(); - verifyBulkExportResults(options, List.of("\"PING1\""), Collections.singletonList("\"PNING3\"")); + verifyBulkExportResults(options, List.of("Patient/PING1"), Collections.singletonList("Patient/PNING3")); myCaptureQueriesListener.logSelectQueries(); ourLog.error("************"); myCaptureQueriesListener.clear(); try { - verifyBulkExportResults(options, List.of("\"PING1\""), Collections.singletonList("\"PNING3\"")); + verifyBulkExportResults(options, List.of("Patient/PING1"), Collections.singletonList("Patient/PNING3")); } finally { myCaptureQueriesListener.logSelectQueries(); @@ -173,12 +182,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { Observation observation = new Observation(); observation.setSubject(new Reference().setReference("Patient/P1")); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); Encounter encounter = new Encounter(); encounter.setSubject(new Reference().setReference("Patient/P1")); encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - String encId = myClient.create().resource(encounter).execute().getId().getIdPart(); + String encId = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); // diff patient patient = new Patient(); @@ -189,16 +198,16 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { observation = new Observation(); observation.setSubject(new Reference().setReference("Patient/P2")); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId2 = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId2 = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); encounter = new Encounter(); encounter.setSubject(new Reference().setReference("Patient/P2")); encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - String encId2 = myClient.create().resource(encounter).execute().getId().getIdPart(); + String encId2 = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); observation = new Observation(); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId3 = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId3 = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); // set the export options BulkDataExportOptions options = new BulkDataExportOptions(); @@ -208,7 +217,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setExportStyle(BulkDataExportOptions.ExportStyle.PATIENT); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, List.of("\"P1\"", "\"" + obsId + "\"", "\"" + encId + "\""), List.of("\"P2\"", "\"" + obsId2 + "\"", "\"" + encId2 + "\"", "\"" + obsId3 + "\"")); + verifyBulkExportResults(options, List.of("Patient/P1", obsId, encId), List.of("Patient/P2", obsId2, encId2, obsId3)); } @Test @@ -223,12 +232,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { Observation observation = new Observation(); observation.setSubject(new Reference().setReference("Patient/P1")); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); Encounter encounter = new Encounter(); encounter.setSubject(new Reference().setReference("Patient/P1")); encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - String encId = myClient.create().resource(encounter).execute().getId().getIdPart(); + String encId = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); // diff patient patient = new Patient(); @@ -239,12 +248,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { observation = new Observation(); observation.setSubject(new Reference().setReference("Patient/P2")); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId2 = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId2 = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); encounter = new Encounter(); encounter.setSubject(new Reference().setReference("Patient/P2")); encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - String encId2 = myClient.create().resource(encounter).execute().getId().getIdPart(); + String encId2 = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); // yet another diff patient patient = new Patient(); @@ -255,12 +264,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { observation = new Observation(); observation.setSubject(new Reference().setReference("Patient/P3")); observation.setStatus(Observation.ObservationStatus.PRELIMINARY); - String obsId3 = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId3 = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); encounter = new Encounter(); encounter.setSubject(new Reference().setReference("Patient/P3")); encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - String encId3 = myClient.create().resource(encounter).execute().getId().getIdPart(); + String encId3 = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); // set the export options BulkDataExportOptions options = new BulkDataExportOptions(); @@ -270,7 +279,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setExportStyle(BulkDataExportOptions.ExportStyle.PATIENT); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, List.of("\"P1\"", "\"" + obsId + "\"", "\"" + encId + "\"", "\"P2\"", "\"" + obsId2 + "\"", "\"" + encId2 + "\""), List.of("\"P3\"", "\"" + obsId3 + "\"", "\"" + encId3 + "\"")); + verifyBulkExportResults(options, List.of("Patient/P1", obsId, encId, "Patient/P2", obsId2, encId2), List.of("Patient/P3", obsId3, encId3)); } @Test @@ -278,23 +287,23 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { // Create some resources Practitioner practitioner = new Practitioner(); practitioner.setActive(true); - String practId = myClient.create().resource(practitioner).execute().getId().getIdPart(); + String practId = myClient.create().resource(practitioner).execute().getId().toUnqualifiedVersionless().getValue(); Organization organization = new Organization(); organization.setActive(true); - String orgId = myClient.create().resource(organization).execute().getId().getIdPart(); + String orgId = myClient.create().resource(organization).execute().getId().toUnqualifiedVersionless().getValue(); organization = new Organization(); organization.setActive(true); - String orgId2 = myClient.create().resource(organization).execute().getId().getIdPart(); + String orgId2 = myClient.create().resource(organization).execute().getId().toUnqualifiedVersionless().getValue(); Location location = new Location(); location.setStatus(Location.LocationStatus.ACTIVE); - String locId = myClient.create().resource(location).execute().getId().getIdPart(); + String locId = myClient.create().resource(location).execute().getId().toUnqualifiedVersionless().getValue(); location = new Location(); location.setStatus(Location.LocationStatus.ACTIVE); - String locId2 = myClient.create().resource(location).execute().getId().getIdPart(); + String locId2 = myClient.create().resource(location).execute().getId().toUnqualifiedVersionless().getValue(); Patient patient = new Patient(); patient.setId("P1"); @@ -310,19 +319,33 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); encounter.setSubject(new Reference("Patient/P1")); Encounter.EncounterParticipantComponent encounterParticipantComponent = new Encounter.EncounterParticipantComponent(); - encounterParticipantComponent.setIndividual(new Reference("Practitioner/" + practId)); + encounterParticipantComponent.setIndividual(new Reference(practId)); encounter.addParticipant(encounterParticipantComponent); Encounter.EncounterLocationComponent encounterLocationComponent = new Encounter.EncounterLocationComponent(); - encounterLocationComponent.setLocation(new Reference("Location/" + locId)); + encounterLocationComponent.setLocation(new Reference(locId)); encounter.addLocation(encounterLocationComponent); - encounter.setServiceProvider(new Reference("Organization/" + orgId)); - String encId = myClient.create().resource(encounter).execute().getId().getIdPart(); + encounter.setServiceProvider(new Reference(orgId)); + String encId = myClient.create().resource(encounter).execute().getId().toUnqualifiedVersionless().getValue(); - encounter = new Encounter(); - encounter.setStatus(Encounter.EncounterStatus.INPROGRESS); - encounter.setSubject(new Reference("Patient/P2")); - encounterLocationComponent.setLocation(new Reference("Location/" + locId2)); - String encId2 = myClient.create().resource(encounter).execute().getId().getIdPart(); + // Second encounter that links to the same location and practitioner + Encounter encounter2 = new Encounter(); + encounter2.setStatus(Encounter.EncounterStatus.INPROGRESS); + encounter2.setSubject(new Reference("Patient/P1")); + Encounter.EncounterParticipantComponent encounterParticipantComponent2 = new Encounter.EncounterParticipantComponent(); + encounterParticipantComponent2.setIndividual(new Reference(practId)); + encounter2.addParticipant(encounterParticipantComponent2); + Encounter.EncounterLocationComponent encounterLocationComponent2 = new Encounter.EncounterLocationComponent(); + encounterLocationComponent2.setLocation(new Reference(locId)); + encounter2.addLocation(encounterLocationComponent2); + encounter2.setServiceProvider(new Reference(orgId)); + String encId2 = myClient.create().resource(encounter2).execute().getId().toUnqualifiedVersionless().getValue(); + + Encounter encounter3 = new Encounter(); + encounter3.setStatus(Encounter.EncounterStatus.INPROGRESS); + encounter3.setSubject(new Reference("Patient/P2")); + Encounter.EncounterLocationComponent encounterLocationComponent3 = encounter3.getLocationFirstRep(); + encounterLocationComponent3.setLocation(new Reference(locId2)); + String encId3 = myClient.create().resource(encounter3).execute().getId().toUnqualifiedVersionless().getValue(); Group group = new Group(); group.setId("Group/G1"); @@ -337,7 +360,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setFilters(new HashSet<>()); options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, List.of("\"P1\"", "\"" + practId + "\"", "\"" + orgId + "\"", "\"" + encId + "\"", "\"" + locId + "\""), List.of("\"P2\"", "\"" + orgId2 + "\"", "\"" + encId2 + "\"", "\"" + locId2 + "\"")); + verifyBulkExportResults(options, List.of("Patient/P1", practId, orgId, encId, encId2, locId), List.of("Patient/P2", orgId2, encId3, locId2)); } @Test @@ -345,15 +368,15 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { // Create some resources Device device = new Device(); device.setStatus(Device.FHIRDeviceStatus.ACTIVE); - String devId = myClient.create().resource(device).execute().getId().getIdPart(); + String devId = myClient.create().resource(device).execute().getId().toUnqualifiedVersionless().getValue(); device = new Device(); device.setStatus(Device.FHIRDeviceStatus.ACTIVE); - String devId2 = myClient.create().resource(device).execute().getId().getIdPart(); + String devId2 = myClient.create().resource(device).execute().getId().toUnqualifiedVersionless().getValue(); device = new Device(); device.setStatus(Device.FHIRDeviceStatus.ACTIVE); - String devId3 = myClient.create().resource(device).execute().getId().getIdPart(); + String devId3 = myClient.create().resource(device).execute().getId().toUnqualifiedVersionless().getValue(); Patient patient = new Patient(); patient.setId("P1"); @@ -367,19 +390,19 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { Observation observation = new Observation(); observation.setStatus(Observation.ObservationStatus.AMENDED); - observation.setDevice(new Reference("Device/" + devId)); + observation.setDevice(new Reference(devId)); observation.setSubject(new Reference("Patient/P1")); - String obsId = myClient.create().resource(observation).execute().getId().getIdPart(); + String obsId = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue(); Provenance provenance = new Provenance(); - provenance.addAgent().setWho(new Reference("Device/" + devId2)); + provenance.addAgent().setWho(new Reference(devId2)); provenance.addTarget(new Reference("Patient/P1")); - String provId = myClient.create().resource(provenance).execute().getId().getIdPart(); + String provId = myClient.create().resource(provenance).execute().getId().toUnqualifiedVersionless().getValue(); provenance = new Provenance(); - provenance.addAgent().setWho(new Reference("Device/" + devId3)); + provenance.addAgent().setWho(new Reference(devId3)); provenance.addTarget(new Reference("Patient/P2")); - String provId2 = myClient.create().resource(provenance).execute().getId().getIdPart(); + String provId2 = myClient.create().resource(provenance).execute().getId().toUnqualifiedVersionless().getValue(); Group group = new Group(); group.setId("Group/G1"); @@ -394,7 +417,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { options.setFilters(new HashSet<>()); options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP); options.setOutputFormat(Constants.CT_FHIR_NDJSON); - verifyBulkExportResults(options, List.of("\"P1\"", "\"" + obsId + "\"", "\"" + provId + "\"", "\"" + devId + "\"", "\"" + devId2 + "\""), List.of("\"P2\"", "\"" + provId2 + "\"", "\"" + devId3 + "\"")); + verifyBulkExportResults(options, List.of("Patient/P1", obsId, provId, devId, devId2), List.of("Patient/P2", provId2, devId3)); } private void verifyBulkExportResults(BulkDataExportOptions theOptions, List theContainedList, List theExcludedList) { @@ -410,21 +433,43 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { // Iterate over the files String report = myJobRunner.getJobInfo(startResponse.getJobId()).getReport(); BulkExportJobResults results = JsonUtil.deserialize(report, BulkExportJobResults.class); - String contents = ""; + + Set foundIds = new HashSet<>(); for (Map.Entry> file : results.getResourceTypeToBinaryIds().entrySet()) { + String resourceType = file.getKey(); List binaryIds = file.getValue(); - assertEquals(1, binaryIds.size()); - Binary binary = myBinaryDao.read(new IdType(binaryIds.get(0))); - assertEquals(Constants.CT_FHIR_NDJSON, binary.getContentType()); - contents += new String(binary.getContent(), Constants.CHARSET_UTF8) + "\n"; - ourLog.info("Next contents for type {} :\n{}", binary.getResourceType(), new String(binary.getContent(), Constants.CHARSET_UTF8)); + for (var nextBinaryId : binaryIds) { + + Binary binary = myBinaryDao.read(new IdType(nextBinaryId)); + assertEquals(Constants.CT_FHIR_NDJSON, binary.getContentType()); + + String nextNdJsonFileContent = new String(binary.getContent(), Constants.CHARSET_UTF8); + try (var iter = new LineIterator(new StringReader(nextNdJsonFileContent))) { + iter.forEachRemaining(t -> { + if (isNotBlank(t)) { + IBaseResource next = myFhirContext.newJsonParser().parseResource(t); + IIdType nextId = next.getIdElement().toUnqualifiedVersionless(); + if (!resourceType.equals(nextId.getResourceType())) { + fail("Found resource of type " + nextId.getResourceType() + " in file for type " + resourceType); + } else { + if (!foundIds.add(nextId.getValue())) { + fail("Found duplicate ID: " + nextId.getValue()); + } + } + } + }); + } catch (IOException e) { + fail(e.toString()); + } + + } } for (String containedString : theContainedList) { - assertThat(contents, Matchers.containsString(containedString)); + assertThat(foundIds, hasItem(containedString)); } for (String excludedString : theExcludedList) { - assertThat(contents, not(Matchers.containsString(excludedString))); + assertThat(foundIds, not(hasItem(excludedString))); } } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java index 75f1a696ce0..522858e1159 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java @@ -36,13 +36,18 @@ import ca.uhn.fhir.jpa.bulk.export.api.IBulkExportProcessor; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.interceptor.ResponseTerminologyTranslationSvc; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; import org.hl7.fhir.instance.model.api.IBaseResource; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import javax.annotation.Nonnull; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.slf4j.LoggerFactory.getLogger; @@ -70,6 +75,7 @@ public class ExpandResourcesStep implements IJobStepWorker allResources = fetchAllResources(idList); @@ -84,17 +90,22 @@ public class ExpandResourcesStep implements IJobStepWorker resources = encodeToString(allResources, jobParameters); + ListMultimap resources = encodeToString(allResources, jobParameters); // set to datasink - BulkExportExpandedResources output = new BulkExportExpandedResources(); - output.setStringifiedResources(resources); - output.setResourceType(idList.getResourceType()); - theDataSink.accept(output); + for (String nextResourceType : resources.keySet()) { - ourLog.info("Expanding of {} resources of type {} completed", - idList.getIds().size(), - idList.getResourceType()); + BulkExportExpandedResources output = new BulkExportExpandedResources(); + output.setStringifiedResources(resources.get(nextResourceType)); + output.setResourceType(nextResourceType); + theDataSink.accept(output); + + ourLog.info("Expanding of {} resources of type {} completed", + idList.getIds().size(), + idList.getResourceType()); + + + } // and return return RunOutcome.SUCCESS; @@ -113,15 +124,19 @@ public class ExpandResourcesStep implements IJobStepWorker encodeToString(List theResources, BulkExportJobParameters theParameters) { + /** + * @return A map - Key is resource type, Value is a collection of serialized resources of that type + */ + private ListMultimap encodeToString(List theResources, BulkExportJobParameters theParameters) { IParser parser = getParser(theParameters); - List resources = new ArrayList<>(); + ListMultimap retVal = ArrayListMultimap.create(); for (IBaseResource resource : theResources) { + String type = myFhirContext.getResourceType(resource); String jsonResource = parser.encodeResourceToString(resource); - resources.add(jsonResource); + retVal.put(type, jsonResource); } - return resources; + return retVal; } private IParser getParser(BulkExportJobParameters theParameters) { diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java index 014ca238eb3..813affb0285 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java @@ -21,6 +21,7 @@ package ca.uhn.fhir.batch2.jobs.export.models; */ import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Multimap; import java.util.List; diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java index 526d19bfe4f..493abb66cce 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java @@ -13,9 +13,6 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.bulk.export.api.IBulkExportProcessor; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.model.api.IQueryParameterType; -import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.interceptor.ResponseTerminologyTranslationSvc; @@ -31,11 +28,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java index 2af3cd15fe1..77a64fd8f21 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java @@ -5,7 +5,6 @@ import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; -import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportBinaryFileId; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; @@ -14,13 +13,12 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; -import ca.uhn.fhir.jpa.bulk.export.api.IBulkExportProcessor; -import ca.uhn.fhir.jpa.bulk.export.model.BulkExportJobStatusEnum; import ca.uhn.fhir.rest.api.server.RequestDetails; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; +import com.google.common.collect.Multimap; import org.hl7.fhir.instance.model.api.IBaseBinary; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.IdType;