honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for bulk export of multiple types (#2797) 

* Add support for bulk export of multiple types

* Add changelog
This commit is contained in:
James Agnew 2021-07-15 14:01:10 -04:00 committed by GitHub
parent 5f2a3af01e
commit 0235296ba7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 311 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2797-allow-multiple-typefilters-in-bulk-export.yaml Normal file
View File

@ -0,0 +1,5 @@
---
type: fix
issue: 2797
title: "When initiating a FHIR bulk export, if more than one `_typeFilter` parameter was supplied
only the first one was respected. This has been corrected."

31
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/provider/BulkDataExportProvider.java
View File

@ -85,7 +85,7 @@ public class BulkDataExportProvider {
@OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
@OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theTypeFilter,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
ServletRequestDetails theRequestDetails
) {
validatePreferAsyncHeader(theRequestDetails);
@ -113,7 +113,7 @@ public class BulkDataExportProvider {
@OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
@OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theTypeFilter,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
@OperationParam(name = JpaConstants.PARAM_EXPORT_MDM, min = 0, max = 1, typeName = "boolean") IPrimitiveType<Boolean> theMdm,
ServletRequestDetails theRequestDetails
) {
@ -151,7 +151,7 @@ public class BulkDataExportProvider {
@OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
@OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theTypeFilter,
@OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
ServletRequestDetails theRequestDetails
) {
validatePreferAsyncHeader(theRequestDetails);
@ -218,11 +218,11 @@ public class BulkDataExportProvider {
}
}
private BulkDataExportOptions buildSystemBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, IPrimitiveType<String> theTypeFilter) {
private BulkDataExportOptions buildSystemBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter) {
return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.SYSTEM);
}
private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, IPrimitiveType<String> theTypeFilter, IIdType theGroupId, IPrimitiveType<Boolean> theExpandMdm) {
private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter, IIdType theGroupId, IPrimitiveType<Boolean> theExpandMdm) {
BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.GROUP);
bulkDataExportOptions.setGroupId(theGroupId);
@ -235,11 +235,11 @@ public class BulkDataExportProvider {
return bulkDataExportOptions;
}
private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, IPrimitiveType<String> theTypeFilter) {
private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter) {
return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.PATIENT);
}
private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, IPrimitiveType<String> theTypeFilter, BulkDataExportOptions.ExportStyle theExportStyle) {
private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter, BulkDataExportOptions.ExportStyle theExportStyle) {
String outputFormat = theOutputFormat != null ? theOutputFormat.getValueAsString() : null;
Set<String> resourceTypes = null;
@ -285,17 +285,22 @@ public class BulkDataExportProvider {
}
}
private Set<String> splitTypeFilters(IPrimitiveType<String> theTypeFilter) {
private Set<String> splitTypeFilters(List<IPrimitiveType<String>> theTypeFilter) {
if (theTypeFilter== null) {
return null;
}
String typeFilterSring = theTypeFilter.getValueAsString();
String[] typeFilters = typeFilterSring.split(FARM_TO_TABLE_TYPE_FILTER_REGEX);
if (typeFilters == null || typeFilters.length == 0) {
return null;
Set<String> retVal = new HashSet<>();
for (IPrimitiveType<String> next : theTypeFilter) {
String typeFilterString = next.getValueAsString();
Arrays
.stream(typeFilterString.split(FARM_TO_TABLE_TYPE_FILTER_REGEX))
.filter(StringUtils::isNotBlank)
.forEach(t->retVal.add(t));
}
return new HashSet<>(Arrays.asList(typeFilters));
return retVal;
}
}

33
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportProviderTest.java
View File

@ -188,6 +188,39 @@ public class BulkDataExportProviderTest {
assertThat(options.getFilters(), containsInAnyOrder("Patient?identifier=foo"));
}
@Test
public void testSuccessfulInitiateBulkRequest_Get_MultipleTypeFilters() throws IOException {
IBulkDataExportSvc.JobInfo jobInfo = new IBulkDataExportSvc.JobInfo()
.setJobId(A_JOB_ID);
when(myBulkDataExportSvc.submitJob(any(),any(), nullable(RequestDetails.class))).thenReturn(jobInfo);
String url = "http://localhost:" + myPort + "/" + JpaConstants.OPERATION_EXPORT
+ "?" + JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT + "=" + UrlUtil.escapeUrlParam(Constants.CT_FHIR_NDJSON)
+ "&" + JpaConstants.PARAM_EXPORT_TYPE + "=" + UrlUtil.escapeUrlParam("Patient,EpisodeOfCare")
+ "&" + JpaConstants.PARAM_EXPORT_TYPE_FILTER + "=" + UrlUtil.escapeUrlParam("Patient?_id=P999999990")
+ "&" + JpaConstants.PARAM_EXPORT_TYPE_FILTER + "=" + UrlUtil.escapeUrlParam("EpisodeOfCare?patient=P999999990");
HttpGet get = new HttpGet(url);
get.addHeader(Constants.HEADER_PREFER, Constants.HEADER_PREFER_RESPOND_ASYNC);
ourLog.info("Request: {}", url);
try (CloseableHttpResponse response = myClient.execute(get)) {
ourLog.info("Response: {}", response.toString());
assertEquals(202, response.getStatusLine().getStatusCode());
assertEquals("Accepted", response.getStatusLine().getReasonPhrase());
assertEquals("http://localhost:" + myPort + "/$export-poll-status?_jobId=" + A_JOB_ID, response.getFirstHeader(Constants.HEADER_CONTENT_LOCATION).getValue());
}
verify(myBulkDataExportSvc, times(1)).submitJob(myBulkDataExportOptionsCaptor.capture(), any(), nullable(RequestDetails.class));
BulkDataExportOptions options = myBulkDataExportOptionsCaptor.getValue();
assertEquals(Constants.CT_FHIR_NDJSON, options.getOutputFormat());
assertThat(options.getResourceTypes(), containsInAnyOrder("Patient", "EpisodeOfCare"));
assertThat(options.getSince(), nullValue());
assertThat(options.getFilters(), containsInAnyOrder("Patient?_id=P999999990", "EpisodeOfCare?patient=P999999990"));
}
@Test
public void testPollForStatus_BUILDING() throws IOException {

60
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java
View File

@ -7,7 +7,6 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
import ca.uhn.fhir.jpa.batch.BatchJobsConfig;
import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobParametersBuilder;
import ca.uhn.fhir.jpa.bulk.export.job.GroupBulkExportJobParametersBuilder;
@ -26,6 +25,7 @@ import ca.uhn.fhir.mdm.api.MdmMatchResultEnum;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import ca.uhn.fhir.util.HapiExtensions;
@ -40,6 +40,7 @@ import org.hl7.fhir.r4.model.Binary;
import org.hl7.fhir.r4.model.CareTeam;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Enumerations;
import org.hl7.fhir.r4.model.EpisodeOfCare;
import org.hl7.fhir.r4.model.Extension;
import org.hl7.fhir.r4.model.Group;
import org.hl7.fhir.r4.model.Immunization;
@ -518,6 +519,63 @@ public class BulkDataExportSvcImplR4Test extends BaseJpaR4Test {
}
}
@Test
public void testGenerateBulkExport_WithMultipleTypeFilters() {
// Create some resources to load
Patient p = new Patient();
p.setId("P999999990");
p.setActive(true);
myPatientDao.update(p);
EpisodeOfCare eoc = new EpisodeOfCare();
eoc.setId("E0");
eoc.getPatient().setReference("Patient/P999999990");
myEpisodeOfCareDao.update(eoc);
// Create a bulk job
HashSet<String> types = Sets.newHashSet("Patient", "EpisodeOfCare");
Set<String> typeFilters = Sets.newHashSet("Patient?_id=P999999990", "EpisodeOfCare?patient=P999999990");
BulkDataExportOptions options = new BulkDataExportOptions();
options.setExportStyle(BulkDataExportOptions.ExportStyle.SYSTEM);
options.setResourceTypes(types);
options.setFilters(typeFilters);
IBulkDataExportSvc.JobInfo jobDetails = myBulkDataExportSvc.submitJob(options);
assertNotNull(jobDetails.getJobId());
// Check the status
IBulkDataExportSvc.JobInfo status = myBulkDataExportSvc.getJobInfoOrThrowResourceNotFound(jobDetails.getJobId());
assertEquals(BulkExportJobStatusEnum.SUBMITTED, status.getStatus());
assertEquals("/$export?_outputFormat=application%2Ffhir%2Bndjson&_type=EpisodeOfCare,Patient&_typeFilter=Patient%3F_id%3DP999999990&_typeFilter=EpisodeOfCare%3Fpatient%3DP999999990", status.getRequest());
// Run a scheduled pass to build the export
myBulkDataExportSvc.buildExportFiles();
awaitAllBulkJobCompletions();
// Fetch the job again
status = myBulkDataExportSvc.getJobInfoOrThrowResourceNotFound(jobDetails.getJobId());
assertEquals(BulkExportJobStatusEnum.COMPLETE, status.getStatus());
assertEquals(2, status.getFiles().size());
// Iterate over the files
for (IBulkDataExportSvc.FileEntry next : status.getFiles()) {
Binary nextBinary = myBinaryDao.read(next.getResourceId());
assertEquals(Constants.CT_FHIR_NDJSON, nextBinary.getContentType());
String nextContents = new String(nextBinary.getContent(), Constants.CHARSET_UTF8);
ourLog.info("Next contents for type {}:\n{}", next.getResourceType(), nextContents);
if ("Patient".equals(next.getResourceType())) {
assertThat(nextContents, containsString("\"id\":\"P999999990\""));
assertEquals(1, nextContents.split("\n").length);
} else if ("EpisodeOfCare".equals(next.getResourceType())) {
assertThat(nextContents, containsString("\"id\":\"E0\""));
assertEquals(1, nextContents.split("\n").length);
} else {
fail(next.getResourceType());
}
}
}
@Test
public void testGenerateBulkExport_WithSince() {

98
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ConceptMapTest.java
View File

@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.BooleanType;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.CodeType;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
@ -27,6 +28,8 @@ import org.springframework.transaction.annotation.Transactional;
import ca.uhn.fhir.rest.api.MethodOutcome;
import java.io.IOException;
public class ResourceProviderR4ConceptMapTest extends BaseResourceProviderR4Test {
private static final Logger ourLog = LoggerFactory.getLogger(ResourceProviderR4ConceptMapTest.class);
@ -171,6 +174,101 @@ public class ResourceProviderR4ConceptMapTest extends BaseResourceProviderR4Test
assertEquals(CM_URL, ((UriType) part.getValue()).getValueAsString());
}
@Test
public void testTranslateByCodeSystemsAndSourceCodeOneToOne_InBatchOperation() {
ConceptMap conceptMap = myConceptMapDao.read(myConceptMapId);
ourLog.info("ConceptMap:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
Bundle bundle = new Bundle();
bundle.setType(Bundle.BundleType.BATCH);
bundle
.addEntry()
.getRequest()
.setMethod(Bundle.HTTPVerb.GET)
.setUrl("ConceptMap/$translate?system=" + CS_URL + "&code=12345" + "&targetsystem=" + CS_URL_2);
ourLog.info("Request:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(bundle));
Bundle respBundle = myClient
.transaction()
.withBundle(bundle)
.execute();
ourLog.info("Response:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(respBundle));
assertEquals(1, respBundle.getEntry().size());
Parameters respParams = (Parameters) respBundle.getEntry().get(0).getResource();
ParametersParameterComponent param = getParameterByName(respParams, "result");
assertTrue(((BooleanType) param.getValue()).booleanValue());
param = getParameterByName(respParams, "message");
assertEquals("Matches found", ((StringType) param.getValue()).getValueAsString());
assertEquals(1, getNumberOfParametersByName(respParams, "match"));
param = getParameterByName(respParams, "match");
assertEquals(3, param.getPart().size());
ParametersParameterComponent part = getPartByName(param, "equivalence");
assertEquals("equal", ((CodeType) part.getValue()).getCode());
part = getPartByName(param, "concept");
Coding coding = (Coding) part.getValue();
assertEquals("34567", coding.getCode());
assertEquals("Target Code 34567", coding.getDisplay());
assertFalse(coding.getUserSelected());
assertEquals(CS_URL_2, coding.getSystem());
assertEquals("Version 2", coding.getVersion());
part = getPartByName(param, "source");
assertEquals(CM_URL, ((UriType) part.getValue()).getValueAsString());
}
@Test
public void testTranslateByCodeSystemsAndSourceCodeOneToOne_InBatchOperation2() throws IOException {
ConceptMap cm = loadResourceFromClasspath(ConceptMap.class, "/r4/conceptmap.json");
myConceptMapDao.update(cm);
Bundle bundle = new Bundle();
bundle.setType(Bundle.BundleType.BATCH);
bundle
.addEntry()
.getRequest()
.setMethod(Bundle.HTTPVerb.GET)
.setUrl("ConceptMap/$translate?url=http://hl7.org/fhir/ConceptMap/CMapHie&system=http://fkcfhir.org/fhir/cs/FMCECCOrderAbbreviation&code=IMed_Janssen&targetsystem=http://fkcfhir.org/fhir/cs/FMCHIEOrderAbbreviation");
ourLog.info("Request:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(bundle));
Bundle respBundle = myClient
.transaction()
.withBundle(bundle)
.execute();
ourLog.info("Response:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(respBundle));
assertEquals(1, respBundle.getEntry().size());
Parameters respParams = (Parameters) respBundle.getEntry().get(0).getResource();
ParametersParameterComponent param = getParameterByName(respParams, "result");
assertTrue(((BooleanType) param.getValue()).booleanValue());
param = getParameterByName(respParams, "message");
assertEquals("Matches found", ((StringType) param.getValue()).getValueAsString());
assertEquals(1, getNumberOfParametersByName(respParams, "match"));
param = getParameterByName(respParams, "match");
assertEquals(3, param.getPart().size());
ParametersParameterComponent part = getPartByName(param, "equivalence");
assertEquals("equivalent", ((CodeType) part.getValue()).getCode());
part = getPartByName(param, "concept");
Coding coding = (Coding) part.getValue();
assertEquals("212", coding.getCode());
assertEquals("COVID-19 Vaccine,vecton-nr,rS-Ad26,PF,0.5mL", coding.getDisplay());
assertFalse(coding.getUserSelected());
assertEquals("http://fkcfhir.org/fhir/cs/FMCHIEOrderAbbreviation", coding.getSystem());
}
@Test
public void testTranslateByCodeSystemsAndSourceCodeUnmapped() {
ConceptMap conceptMap = myConceptMapDao.read(myConceptMapId);

98
hapi-fhir-jpaserver-base/src/test/resources/r4/conceptmap.json Normal file
View File

@ -0,0 +1,98 @@
{
"resourceType": "ConceptMap",
"id": "CMapHie",
"meta": {
"extension": [
{
"url": "http://hapifhir.io/fhir/StructureDefinition/resource-meta-source",
"valueUri": "#VAL8lnninHkvaEWc"
}
],
"versionId": "1",
"lastUpdated": "2021-07-08T14:19:11.748-04:00"
},
"url": "http://hl7.org/fhir/ConceptMap/CMapHie",
"identifier": {
"system": "urn:ietf:rfc:3986",
"value": "urn:uuid:53cd62ee-033e-414c-9f58-3ca97b5ffc3b"
},
"version": "4.0.1",
"name": "FHIR-v3-Address-Use",
"title": "FHIR/v3 Address Use Mapping",
"status": "draft",
"experimental": true,
"date": "2012-06-13",
"publisher": "HL7, Inc",
"contact": [
{
"name": "FHIR project team (example)",
"telecom": [
{
"system": "url",
"value": "http://hl7.org/fhir"
}
]
}
],
"description": "A mapping between the ECC and HIE Code systems",
"useContext": [
{
"code": {
"system": "http://terminology.hl7.org/CodeSystem/usage-context-type",
"code": "venue"
},
"valueCodeableConcept": {
"text": "for CCDA Usage"
}
}
],
"jurisdiction": [
{
"coding": [
{
"system": "urn:iso:std:iso:3166",
"code": "US"
}
]
}
],
"purpose": "To help implementers map from HL7 v3/CDA to FHIR",
"copyright": "Creative Commons 0",
"sourceUri": "http://fkcfhir.org/fhir/vs/FMCOrderAbbreviation",
"targetUri": "http://fkcfhir.org/fhir/vs/FMCHIEAbbreviation",
"group": [
{
"source": "http://fkcfhir.org/fhir/cs/FMCECCOrderAbbreviation",
"target": "http://fkcfhir.org/fhir/cs/FMCHIEOrderAbbreviation",
"element": [
{
"code": "IMed_Janssen",
"display": "COVID-19 Vaccine-Janssen",
"target": [
{
"code": "212",
"display": "COVID-19 Vaccine,vecton-nr,rS-Ad26,PF,0.5mL",
"equivalence": "equivalent"
}
]
},
{
"code": "IMed_Moderna1",
"display": "COVID-19 Vaccine-Moderna (Dose 1 of 2)",
"target": [
{
"code": "207",
"display": "COVID-19, mRNA,LNP-S,PF,100 mcg/0.5 mL dose",
"equivalence": "equivalent"
}
]
}
],
"unmapped": {
"mode": "fixed",
"code": "unknown",
"display": "unknown"
}
}
]
}