Group bulk export _since parameter issue (#4389)

* Test + fix

* javadoc

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4387-group-bulk-export-since-parameter-error.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 4387
+title: "Previously, when use group bulk export, if the group was referenced to a patient resource and the patient has not been updated after the _since parameter of the bulk export, none of their data/sub-resources comes through even if the sub-resources were qualified (By qualified, I mean the sub-resources were upadted/created after _since parameter).
+Now, this issue has been fixed."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java

@@ -149,7 +149,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 		Set<String> patientSearchParams = SearchParameterUtil.getPatientSearchParamsForResourceType(myContext, theParams.getResourceType());
 
 		for (String patientSearchParam : patientSearchParams) {
-			List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParams);
+			List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParams, false);
 			for (SearchParameterMap map : maps) {
 				//Ensure users did not monkey with the patient compartment search parameter.
 				validateSearchParametersForPatient(map, theParams);
@@ -196,7 +196,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 	private LinkedHashSet<JpaPid> getPidsForSystemStyleExport(ExportPIDIteratorParameters theParams, String theJobId, RuntimeResourceDefinition theDef) {
 		LinkedHashSet<JpaPid> pids = new LinkedHashSet<>();
 		// System
-		List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(theDef, theParams);
+		List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(theDef, theParams, true);
 		ISearchBuilder<JpaPid> searchBuilder = getSearchBuilderForResourceType(theParams.getResourceType());
 
 		for (SearchParameterMap map : maps) {
@@ -311,7 +311,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 	 * possibly expanded by MDM, and don't have to go and fetch other resource DAOs.
 	 */
 	private LinkedHashSet<JpaPid> getExpandedPatientList(ExportPIDIteratorParameters theParameters) {
-		List<JpaPid> members = getMembersFromGroupWithFilter(theParameters);
+		List<JpaPid> members = getMembersFromGroupWithFilter(theParameters, true);
 		List<IIdType> ids = members.stream().map(member -> new IdDt("Patient/" + member)).collect(Collectors.toList());
 		ourLog.info("While extracting patients from a group, we found {} patients.", ids.size());
 		ourLog.info("Found patients: {}", ids.stream().map(id -> id.getValue()).collect(Collectors.joining(", ")));
@@ -339,12 +339,11 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 	 *
 	 * @return A list of strings representing the Patient IDs of the members (e.g. ["P1", "P2", "P3"]
 	 */
-	private List<JpaPid> getMembersFromGroupWithFilter(ExportPIDIteratorParameters theParameters) {
+	private List<JpaPid> getMembersFromGroupWithFilter(ExportPIDIteratorParameters theParameters, boolean theConsiderSince) {
 		RuntimeResourceDefinition def = myContext.getResourceDefinition("Patient");
-		List<String> pids = new ArrayList<>();
 		List<JpaPid> resPids = new ArrayList<>();
 
-		List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParameters);
+		List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParameters, theConsiderSince);
 
 		maps.forEach(map -> addMembershipToGroupClause(map, theParameters.getGroupId()));
 
@@ -432,7 +431,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 
 		//Build SP map
 		//First, inject the _typeFilters and _since from the export job
-		List<SearchParameterMap> expandedSpMaps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(theDef, theParams);
+		List<SearchParameterMap> expandedSpMaps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(theDef, theParams, true);
 		for (SearchParameterMap expandedSpMap : expandedSpMaps) {
 
 			//Since we are in a bulk job, we have to ensure the user didn't jam in a patient search param, since we need to manually set that.
@@ -520,7 +519,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 
 		//Now manually add the members of the group (its possible even with mdm expansion that some members dont have MDM matches,
 		//so would be otherwise skipped
-		List<JpaPid> membersFromGroupWithFilter = getMembersFromGroupWithFilter(theParams);
+		List<JpaPid> membersFromGroupWithFilter = getMembersFromGroupWithFilter(theParams, false);
 		ourLog.debug("Group with ID [{}] has been expanded to: {}", theParams.getGroupId(), membersFromGroupWithFilter);
 		expandedIds.addAll(membersFromGroupWithFilter);
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java

@@ -184,7 +184,7 @@ public class JpaBulkExportProcessorTest {
 		// when
 		when(myDaoConfig.getIndexMissingFields())
 			.thenReturn(DaoConfig.IndexEnabledEnum.ENABLED);
-		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters)))
+		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters), any(boolean.class)))
 			.thenReturn(maps);
 		// from getSearchBuilderForLocalResourceType
 		when(myDaoRegistry.getResourceDao(anyString()))
@@ -259,7 +259,7 @@ public class JpaBulkExportProcessorTest {
 		ISearchBuilder searchBuilder = mock(ISearchBuilder.class);
 
 		// from getMembersFromGroupWithFilter
-		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters)))
+		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters), any(boolean.class)))
 			.thenReturn(Collections.singletonList(new SearchParameterMap()));
 		// from getSearchBuilderForLocalResourceType
 		when(myDaoRegistry.getResourceDao(not(eq("Group"))))
@@ -355,7 +355,7 @@ public class JpaBulkExportProcessorTest {
 			.thenReturn(groupResource);
 		when(myIdHelperService.getPidOrNull(any(), eq(groupResource)))
 			.thenReturn(groupId);
-		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters)))
+		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters), any(boolean.class)))
 			.thenReturn(Collections.singletonList(new SearchParameterMap()));
 		when(myDaoRegistry.getResourceDao(not(eq("Group"))))
 			.thenReturn(mockDao);
@@ -412,7 +412,8 @@ public class JpaBulkExportProcessorTest {
 		// when
 		when(myBulkExportHelperService.createSearchParameterMapsForResourceType(
 			any(RuntimeResourceDefinition.class),
-			any(ExportPIDIteratorParameters.class)
+			any(ExportPIDIteratorParameters.class),
+			any(boolean.class)
 		)).thenReturn(Collections.singletonList(new SearchParameterMap()));
 		when(myDaoRegistry.getResourceDao(eq("Patient")))
 			.thenReturn(dao);

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java

@@ -22,6 +22,7 @@ import org.hl7.fhir.r4.model.Encounter;
 import org.hl7.fhir.r4.model.Enumerations;
 import org.hl7.fhir.r4.model.Group;
 import org.hl7.fhir.r4.model.IdType;
+import org.hl7.fhir.r4.model.InstantType;
 import org.hl7.fhir.r4.model.Location;
 import org.hl7.fhir.r4.model.MedicationAdministration;
 import org.hl7.fhir.r4.model.Observation;
@@ -43,6 +44,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -469,6 +471,46 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 		verifyBulkExportResults(options, List.of("Patient/P1", obsId, provId, devId, devId2), List.of("Patient/P2", provId2, devId3));
 	}
 
+	@Test
+	public void testGroupBulkExport_QualifiedSubResourcesOfUnqualifiedPatientShouldShowUp() throws InterruptedException {
+
+		// Patient with lastUpdated before _since
+		Patient patient = new Patient();
+		patient.setId("A");
+		myClient.update().resource(patient).execute();
+
+		// Sleep for 1 sec
+		ourLog.info("Patient lastUpdated: " + InstantType.withCurrentTime().getValueAsString());
+		Thread.sleep(1000);
+
+		// LastUpdated since now
+		Date timeDate = InstantType.now().getValue();
+		ourLog.info(timeDate.toString());
+
+		// Group references to Patient/A
+		Group group = new Group();
+		group.setId("B");
+		group.addMember().getEntity().setReference("Patient/A");
+		myClient.update().resource(group).execute();
+
+		// Observation references to Patient/A
+		Observation observation = new Observation();
+		observation.setId("C");
+		observation.setSubject(new Reference("Patient/A"));
+		myClient.update().resource(observation).execute();
+
+		// Set the export options
+		BulkDataExportOptions options = new BulkDataExportOptions();
+		options.setResourceTypes(Sets.newHashSet("Patient", "Observation", "Group"));
+		options.setGroupId(new IdType("Group", "B"));
+		options.setFilters(new HashSet<>());
+		options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP);
+		options.setSince(timeDate);
+		options.setOutputFormat(Constants.CT_FHIR_NDJSON);
+		// Should get the sub-resource (Observation) even the patient hasn't been updated after the _since param
+		verifyBulkExportResults(options, List.of("Observation/C", "Group/B"), List.of("Patient/A"));
+	}
+
 	@Test
 	public void testPatientBulkExportWithReferenceToAuthor_ShouldShowUp() {
 		myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED);

hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/BulkExportHelperService.java

@@ -47,7 +47,12 @@ public class BulkExportHelperService {
 	@Autowired
 	private FhirContext myContext;
 
-	public List<SearchParameterMap> createSearchParameterMapsForResourceType(RuntimeResourceDefinition theDef, ExportPIDIteratorParameters theParams) {
+	/**
+	 * Given the parameters, create the search parameter map based on type filters and the _since parameter.
+	 *
+	 * The input boolean theConsiderSince determines whether to consider the lastUpdated date in the search parameter map.
+	 */
+	public List<SearchParameterMap> createSearchParameterMapsForResourceType(RuntimeResourceDefinition theDef, ExportPIDIteratorParameters theParams, boolean theConsiderSince) {
 		String resourceType = theDef.getName();
 		List<String> typeFilters = theParams.getFilters();
 		List<SearchParameterMap> spMaps = null;
@@ -63,7 +68,9 @@ public class BulkExportHelperService {
 		//None of the _typeFilters applied to the current resource type, so just make a simple one.
 		if (spMaps.isEmpty()) {
 			SearchParameterMap defaultMap = new SearchParameterMap();
+			if (theConsiderSince) {
 				enhanceSearchParameterMapWithCommonParameters(defaultMap, theParams.getStartDate());
+			}
 			spMaps = Collections.singletonList(defaultMap);
 		}