honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Apply fixes to review comments for #4772 (#4780) 

* Add post-fetch filtering to bulk export

* Add changelog

* Add validator

* Test fixes

* Test fix

* Review comments for #4772

* Commit fix
This commit is contained in:
James Agnew 2023-04-27 10:23:01 -04:00 committed by GitHub
parent f6094eed47
commit f9de5c918e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4772-add-post-fetch-filtering-to-bulk-export.yaml
View File

@ -4,6 +4,6 @@ issue: 4772
title: "A new optional parameter called `_typePostFetchFilterUrl` has been added to the
Bulk Export `$export`
operation parameters. This parameter allows filters to be specified that will be applied
in-memory to the resources after that have been initially fetched by the database. This can
in-memory to the resources after they have been initially fetched by the database. This can
be used to allow complex filters which only remove small numbers of resources to be
efficiently applied against large datasets."

56
hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java
View File

@ -136,19 +136,17 @@ public class ExpandResourcesStep implements IJobStepWorker<BulkExportJobParamete
ListMultimap<String, String> resources = encodeToString(allResources, parameters);
// set to datasink
if (!resources.isEmpty()) {
for (String nextResourceType : resources.keySet()) {
for (String nextResourceType : resources.keySet()) {
ExpandedResourcesList output = new ExpandedResourcesList();
output.setStringifiedResources(resources.get(nextResourceType));
output.setResourceType(nextResourceType);
theDataSink.accept(output);
ExpandedResourcesList output = new ExpandedResourcesList();
output.setStringifiedResources(resources.get(nextResourceType));
output.setResourceType(nextResourceType);
theDataSink.accept(output);
ourLog.info("Expanding of {} resources of type {} completed",
idList.getIds().size(),
idList.getResourceType());
ourLog.info("Expanding of {} resources of type {} completed",
idList.getIds().size(),
idList.getResourceType());
}
}
// and return
@ -158,25 +156,7 @@ public class ExpandResourcesStep implements IJobStepWorker<BulkExportJobParamete
private void applyPostFetchFiltering(List<IBaseResource> theResources, List<String> thePostFetchFilterUrls, String theInstanceId, String theChunkId) {
int numRemoved = 0;
for (Iterator<IBaseResource> iter = theResources.iterator(); iter.hasNext(); ) {
IBaseResource nextResource = iter.next();
String nextResourceType = myFhirContext.getResourceType(nextResource);
boolean matched = false;
for (String nextPostFetchFilterUrl : thePostFetchFilterUrls) {
// TODO: JA in next ticket - Add validation to the filter URLs when the job is submitted
// We should make sure that the format is [resourceType]?[at least one param] and that
// the param can be evaluated by the in memory matcher
if (nextPostFetchFilterUrl.contains("?")) {
String resourceType = nextPostFetchFilterUrl.substring(0, nextPostFetchFilterUrl.indexOf('?'));
if (nextResourceType.equals(resourceType)) {
InMemoryMatchResult matchResult = myInMemoryResourceMatcher.match(nextPostFetchFilterUrl, nextResource, null, new SystemRequestDetails());
if (matchResult.matched()) {
matched = true;
break;
}
}
}
}
boolean matched = applyPostFetchFilteringForSingleResource(thePostFetchFilterUrls, iter);
if (!matched) {
iter.remove();
@ -189,6 +169,24 @@ public class ExpandResourcesStep implements IJobStepWorker<BulkExportJobParamete
}
}
private boolean applyPostFetchFilteringForSingleResource(List<String> thePostFetchFilterUrls, Iterator<IBaseResource> iter) {
IBaseResource nextResource = iter.next();
String nextResourceType = myFhirContext.getResourceType(nextResource);
for (String nextPostFetchFilterUrl : thePostFetchFilterUrls) {
if (nextPostFetchFilterUrl.contains("?")) {
String resourceType = nextPostFetchFilterUrl.substring(0, nextPostFetchFilterUrl.indexOf('?'));
if (nextResourceType.equals(resourceType)) {
InMemoryMatchResult matchResult = myInMemoryResourceMatcher.match(nextPostFetchFilterUrl, nextResource, null, new SystemRequestDetails());
if (matchResult.matched()) {
return true;
}
}
}
}
return false;
}
private List<IBaseResource> fetchAllResources(ResourceIdList theIds, RequestPartitionId theRequestPartitionId) {
ArrayListMultimap<String, String> typeToIds = ArrayListMultimap.create();
theIds.getIds().forEach(t -> typeToIds.put(t.getResourceType(), t.getId()));

3
hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java
View File

@ -48,6 +48,9 @@ public class BulkExportParameters extends Batch2BaseJobParameters {
*/
private List<String> myFilters;
/**
* URLs to be applied by the inMemoryMatcher after the SQL select
*/
private List<String> myPostFetchFilterUrls;
/**