diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4772-add-post-fetch-filtering-to-bulk-export.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4772-add-post-fetch-filtering-to-bulk-export.yaml index 3bd0e2051de..1efc1254c25 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4772-add-post-fetch-filtering-to-bulk-export.yaml +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4772-add-post-fetch-filtering-to-bulk-export.yaml @@ -4,6 +4,6 @@ issue: 4772 title: "A new optional parameter called `_typePostFetchFilterUrl` has been added to the Bulk Export `$export` operation parameters. This parameter allows filters to be specified that will be applied - in-memory to the resources after that have been initially fetched by the database. This can + in-memory to the resources after they have been initially fetched by the database. This can be used to allow complex filters which only remove small numbers of resources to be efficiently applied against large datasets." diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java index 3728a274a2c..e1bd48781df 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java @@ -136,19 +136,17 @@ public class ExpandResourcesStep implements IJobStepWorker resources = encodeToString(allResources, parameters); // set to datasink - if (!resources.isEmpty()) { - for (String nextResourceType : resources.keySet()) { + for (String nextResourceType : resources.keySet()) { - ExpandedResourcesList output = new ExpandedResourcesList(); - output.setStringifiedResources(resources.get(nextResourceType)); - output.setResourceType(nextResourceType); - theDataSink.accept(output); + ExpandedResourcesList output = new ExpandedResourcesList(); + output.setStringifiedResources(resources.get(nextResourceType)); + output.setResourceType(nextResourceType); + theDataSink.accept(output); - ourLog.info("Expanding of {} resources of type {} completed", - idList.getIds().size(), - idList.getResourceType()); + ourLog.info("Expanding of {} resources of type {} completed", + idList.getIds().size(), + idList.getResourceType()); - } } // and return @@ -158,25 +156,7 @@ public class ExpandResourcesStep implements IJobStepWorker theResources, List thePostFetchFilterUrls, String theInstanceId, String theChunkId) { int numRemoved = 0; for (Iterator iter = theResources.iterator(); iter.hasNext(); ) { - IBaseResource nextResource = iter.next(); - String nextResourceType = myFhirContext.getResourceType(nextResource); - boolean matched = false; - - for (String nextPostFetchFilterUrl : thePostFetchFilterUrls) { - // TODO: JA in next ticket - Add validation to the filter URLs when the job is submitted - // We should make sure that the format is [resourceType]?[at least one param] and that - // the param can be evaluated by the in memory matcher - if (nextPostFetchFilterUrl.contains("?")) { - String resourceType = nextPostFetchFilterUrl.substring(0, nextPostFetchFilterUrl.indexOf('?')); - if (nextResourceType.equals(resourceType)) { - InMemoryMatchResult matchResult = myInMemoryResourceMatcher.match(nextPostFetchFilterUrl, nextResource, null, new SystemRequestDetails()); - if (matchResult.matched()) { - matched = true; - break; - } - } - } - } + boolean matched = applyPostFetchFilteringForSingleResource(thePostFetchFilterUrls, iter); if (!matched) { iter.remove(); @@ -189,6 +169,24 @@ public class ExpandResourcesStep implements IJobStepWorker thePostFetchFilterUrls, Iterator iter) { + IBaseResource nextResource = iter.next(); + String nextResourceType = myFhirContext.getResourceType(nextResource); + + for (String nextPostFetchFilterUrl : thePostFetchFilterUrls) { + if (nextPostFetchFilterUrl.contains("?")) { + String resourceType = nextPostFetchFilterUrl.substring(0, nextPostFetchFilterUrl.indexOf('?')); + if (nextResourceType.equals(resourceType)) { + InMemoryMatchResult matchResult = myInMemoryResourceMatcher.match(nextPostFetchFilterUrl, nextResource, null, new SystemRequestDetails()); + if (matchResult.matched()) { + return true; + } + } + } + } + return false; + } + private List fetchAllResources(ResourceIdList theIds, RequestPartitionId theRequestPartitionId) { ArrayListMultimap typeToIds = ArrayListMultimap.create(); theIds.getIds().forEach(t -> typeToIds.put(t.getResourceType(), t.getId())); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java index 16882ceb1f2..6151c2340ee 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java @@ -48,6 +48,9 @@ public class BulkExportParameters extends Batch2BaseJobParameters { */ private List myFilters; + /** + * URLs to be applied by the inMemoryMatcher after the SQL select + */ private List myPostFetchFilterUrls; /**