From cc1861e87691f3539c3e7b2ff4ce7613ac547c8a Mon Sep 17 00:00:00 2001 From: Tadgh Date: Thu, 25 Mar 2021 15:48:03 -0400 Subject: [PATCH] Update test. --- .../GoldenResourceAnnotatingProcessor.java | 65 ++++++++++++++----- .../jpa/bulk/job/BulkExportJobConfig.java | 2 +- .../jpa/bulk/job/GroupBulkItemReader.java | 45 ++++++------- .../jpa/dao/mdm/MdmExpansionCacheSvc.java | 12 +++- .../jpa/bulk/BulkDataExportSvcImplR4Test.java | 17 +++-- 5 files changed, 95 insertions(+), 46 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/processors/GoldenResourceAnnotatingProcessor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/processors/GoldenResourceAnnotatingProcessor.java index c578a71b34f..70d732e3326 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/processors/GoldenResourceAnnotatingProcessor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/processors/GoldenResourceAnnotatingProcessor.java @@ -24,13 +24,14 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.fhirpath.IFhirPath; import ca.uhn.fhir.jpa.batch.log.Logs; +import ca.uhn.fhir.jpa.bulk.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.dao.mdm.MdmExpansionCacheSvc; import ca.uhn.fhir.util.ExtensionUtil; import ca.uhn.fhir.util.SearchParameterUtil; -import org.hl7.fhir.instance.model.api.IBase; +import org.apache.commons.lang3.StringUtils; +import org.hl7.fhir.instance.model.api.IBaseExtension; import org.hl7.fhir.instance.model.api.IBaseReference; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; import org.slf4j.Logger; import org.springframework.batch.item.ItemProcessor; import org.springframework.beans.factory.annotation.Autowired; @@ -55,7 +56,11 @@ public class GoldenResourceAnnotatingProcessor implements ItemProcessor process(List theIBaseResources) throws Exception { - if (myRuntimeSearchParam == null) { - myRuntimeSearchParam = SearchParameterUtil.getPatientSearchParamForResourceType(myContext, myResourceType); + if (myMdmEnabled) { + if (myRuntimeSearchParam == null) { + myRuntimeSearchParam = SearchParameterUtil.getPatientSearchParamForResourceType(myContext, myResourceType); + } + String path = runtimeSearchParamFhirPath(); + theIBaseResources.forEach(iBaseResource -> annotateClinicalResourceWithRelatedGoldenResourcePatient(path, iBaseResource)); } - String path = runtimeSearchParamFhirPath(); - theIBaseResources - .forEach(iBaseResource -> annotateResourceWithRelatedGoldenResourcePatient(path, iBaseResource)); + return theIBaseResources; } - private void annotateResourceWithRelatedGoldenResourcePatient(String path, IBaseResource iBaseResource) { - Optional evaluate = getFhirParser().evaluateFirst(iBaseResource, path, IBaseReference.class); - if (evaluate.isPresent()) { - String sourceResourceId = evaluate.get().getReferenceElement().getIdPart(); - ExtensionUtil.setExtension(myContext, iBaseResource, ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL, myMdmExpansionCacheSvc.getGoldenResourceId(sourceResourceId)); + private void annotateClinicalResourceWithRelatedGoldenResourcePatient(String path, IBaseResource iBaseResource) { + Optional patientReference = getPatientReference(path, iBaseResource); + if (patientReference.isPresent()) { + addGoldenResourceExtension(iBaseResource, patientReference.get()); } else { - ourLog.warn("Failed to find the patient compartment information for resource {}", iBaseResource); + ourLog.warn("Failed to find the patient reference information for resource {}", iBaseResource); } } + private Optional getPatientReference(String path, IBaseResource iBaseResource) { + //In the case of patient, we will just use the raw ID. + if (myResourceType.equalsIgnoreCase("Patient")) { + return Optional.of(iBaseResource.getIdElement().getIdPart()); + //Otherwise, we will perform evaluation of the fhirPath. + } else { + Optional optionalReference = getFhirParser().evaluateFirst(iBaseResource, path, IBaseReference.class); + return optionalReference.map(theIBaseReference -> theIBaseReference.getReferenceElement().getIdPart()); + } + } + + private void addGoldenResourceExtension(IBaseResource iBaseResource, String sourceResourceId) { + String goldenResourceId = myMdmExpansionCacheSvc.getGoldenResourceId(sourceResourceId); + if (!StringUtils.isBlank(goldenResourceId)) { + IBaseExtension extension = ExtensionUtil.getOrCreateExtension(iBaseResource, ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL); + ExtensionUtil.setExtension(myContext, extension, "reference", prefixPatient(goldenResourceId)); + } + } + + private String prefixPatient(String theResourceId) { + return "Patient/" + theResourceId; + } + private IFhirPath getFhirParser() { if (myFhirPath == null) { myFhirPath = myContext.newFhirPath(); @@ -90,9 +119,15 @@ public class GoldenResourceAnnotatingProcessor implements ItemProcessor> lists = myMdmLinkDao.expandPidsFromGroupPidGivenMatchResult(pidOrNull, MdmMatchResultEnum.MATCH); - lists.forEach(patientPidsToExport::addAll); } List resourcePersistentIds = patientPidsToExport @@ -161,11 +159,6 @@ public class GroupBulkItemReader extends BaseBulkItemReader implements ItemReade if (myMdmEnabled) { List> goldenPidTargetPidTuples = myMdmLinkDao.expandPidsFromGroupPidGivenMatchResult(pidOrNull, MdmMatchResultEnum.MATCH); //Now lets translate these pids into resource IDs - - System.out.println("ZOOPER"); - goldenPidTargetPidTuples.stream() - .map(Object::toString) - .forEach(list -> System.out.println(String.join(", ", list))); Set uniquePids = new HashSet<>(); goldenPidTargetPidTuples.forEach(uniquePids::addAll); Map> pidToForcedIdMap = myIdHelperService.translatePidsToForcedIds(uniquePids); @@ -180,22 +173,8 @@ public class GroupBulkItemReader extends BaseBulkItemReader implements ItemReade } goldenResourceToSourcePidMap.get(goldenPid).add(sourcePid); } - Map sourceResourceIdToGoldenResourceIdMap = new HashMap<>(); + populateMdmResourceCacheIfNeeded(goldenResourceToSourcePidMap); - goldenResourceToSourcePidMap.entrySet() - .forEach(entry -> { - Long key = entry.getKey(); - String goldenResourceId = myIdHelperService.translatePidIdToForcedId(new ResourcePersistentId(key)).orElse(key.toString()); - - Map> pidsToForcedIds = myIdHelperService.translatePidsToForcedIds(entry.getValue()); - Set sourceResourceIds = pidsToForcedIds.entrySet().stream() - .map(ent -> ent.getValue().isPresent() ? ent.getValue().get() : ent.getKey().toString()) - .collect(Collectors.toSet()); - - sourceResourceIds - .forEach(sourceResourceId -> sourceResourceIdToGoldenResourceIdMap.put(sourceResourceId, goldenResourceId)); - }); - myMdmExpansionCacheSvc.setCacheContents(sourceResourceIdToGoldenResourceIdMap); //If the result of the translation is an empty optional, it means there is no forced id, and we can use the PID as the resource ID. Set resolvedResourceIds = pidToForcedIdMap.entrySet().stream() @@ -212,6 +191,28 @@ public class GroupBulkItemReader extends BaseBulkItemReader implements ItemReade return expandedIds; } + private void populateMdmResourceCacheIfNeeded(Map> goldenResourceToSourcePidMap) { + if (myMdmExpansionCacheSvc.hasBeenPopulated()) { + return; + } + Map sourceResourceIdToGoldenResourceIdMap = new HashMap<>(); + + goldenResourceToSourcePidMap.entrySet() + .forEach(entry -> { + Long key = entry.getKey(); + String goldenResourceId = myIdHelperService.translatePidIdToForcedId(new ResourcePersistentId(key)).orElse(key.toString()); + + Map> pidsToForcedIds = myIdHelperService.translatePidsToForcedIds(entry.getValue()); + Set sourceResourceIds = pidsToForcedIds.entrySet().stream() + .map(ent -> ent.getValue().isPresent() ? ent.getValue().get() : ent.getKey().toString()) + .collect(Collectors.toSet()); + + sourceResourceIds + .forEach(sourceResourceId -> sourceResourceIdToGoldenResourceIdMap.put(sourceResourceId, goldenResourceId)); + }); + myMdmExpansionCacheSvc.setCacheContents(sourceResourceIdToGoldenResourceIdMap); + } + private void queryResourceTypeWithReferencesToPatients(Set myReadPids, List idChunk) { //Build SP map //First, inject the _typeFilters and _since from the export job diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmExpansionCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmExpansionCacheSvc.java index b923cc282a2..6e80ba7e58b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmExpansionCacheSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmExpansionCacheSvc.java @@ -2,15 +2,15 @@ package ca.uhn.fhir.jpa.dao.mdm; import org.slf4j.Logger; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static org.slf4j.LoggerFactory.getLogger; public class MdmExpansionCacheSvc { private static final Logger ourLog = getLogger(MdmExpansionCacheSvc.class); - private Map mySourceToGoldenIdCache = new HashMap<>(); + private ConcurrentHashMap mySourceToGoldenIdCache = new ConcurrentHashMap<>(); public String getGoldenResourceId(String theSourceId) { ourLog.info(buildLog("About to lookup cached resource ID " + theSourceId, true)); @@ -32,6 +32,12 @@ public class MdmExpansionCacheSvc { } public void setCacheContents(Map theSourceResourceIdToGoldenResourceIdMap) { - this.mySourceToGoldenIdCache = theSourceResourceIdToGoldenResourceIdMap; + if (mySourceToGoldenIdCache.isEmpty()) { + this.mySourceToGoldenIdCache.putAll(theSourceResourceIdToGoldenResourceIdMap); + } + } + + public boolean hasBeenPopulated() { + return !mySourceToGoldenIdCache.isEmpty(); } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java index 86d78abd8ae..0e1ea8e3453 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; import ca.uhn.fhir.jpa.batch.BatchJobsConfig; import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; +import ca.uhn.fhir.jpa.batch.processors.GoldenResourceAnnotatingProcessor; import ca.uhn.fhir.jpa.bulk.api.BulkDataExportOptions; import ca.uhn.fhir.jpa.bulk.api.IBulkDataExportSvc; import ca.uhn.fhir.jpa.bulk.job.BulkExportJobParametersBuilder; @@ -646,7 +647,7 @@ public class BulkDataExportSvcImplR4Test extends BaseJpaR4Test { // Create a bulk job BulkDataExportOptions bulkDataExportOptions = new BulkDataExportOptions(); bulkDataExportOptions.setOutputFormat(null); - bulkDataExportOptions.setResourceTypes(Sets.newHashSet("Immunization")); + bulkDataExportOptions.setResourceTypes(Sets.newHashSet("Immunization", "Observation", "Patient")); bulkDataExportOptions.setSince(null); bulkDataExportOptions.setFilters(null); bulkDataExportOptions.setGroupId(myPatientGroupId); @@ -654,22 +655,28 @@ public class BulkDataExportSvcImplR4Test extends BaseJpaR4Test { bulkDataExportOptions.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP); IBulkDataExportSvc.JobInfo jobDetails = myBulkDataExportSvc.submitJob(bulkDataExportOptions); - myBulkDataExportSvc.buildExportFiles(); awaitAllBulkJobCompletions(); IBulkDataExportSvc.JobInfo jobInfo = myBulkDataExportSvc.getJobInfoOrThrowResourceNotFound(jobDetails.getJobId()); assertThat(jobInfo.getStatus(), equalTo(BulkJobStatusEnum.COMPLETE)); - assertThat(jobInfo.getFiles().size(), equalTo(1)); + assertThat(jobInfo.getFiles().size(), equalTo(3)); assertThat(jobInfo.getFiles().get(0).getResourceType(), is(equalTo("Immunization"))); // Iterate over the files + String nextContents = getBinaryContents(jobInfo, 0); - assertThat(jobInfo.getFiles().get(0).getResourceType(), is(equalTo("Immunization"))); + assertThat(nextContents, is(containsString(GoldenResourceAnnotatingProcessor.ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL))); - assertThat(nextContents, is(containsString("subject_golden_resource"))); + nextContents = getBinaryContents(jobInfo, 1); + assertThat(jobInfo.getFiles().get(1).getResourceType(), is(equalTo("Observation"))); + assertThat(nextContents, is(containsString(GoldenResourceAnnotatingProcessor.ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL))); + + nextContents = getBinaryContents(jobInfo, 2); + assertThat(jobInfo.getFiles().get(2).getResourceType(), is(equalTo("Patient"))); + assertThat(nextContents, is(containsString(GoldenResourceAnnotatingProcessor.ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL))); }