5339 bulk export errors when patient compartment searchparameter of the resource is not present (#5340)

* added failing test

* implemented solution, and fixed test

* added changelog

* fixed formatting

* fixed test not passing

* fixed test and formatting

* added warning for failing the active search param check

* code review change

---------

Co-authored-by: Steven Li <steven@smilecdr.com>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_0_0/5339-bulk-export-errors-when-patient-compartment-searchparameter-of-the-resource-is-not-present.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 5339
+jira: SMILE-7236
+title: "Previously, Bulk Export will error when processing a resource if the patient compartment SearchParameter of that resource is not present.
+This has been fixed, the new behaviour is to ignore such resources."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java

@@ -53,6 +53,7 @@ import ca.uhn.fhir.rest.param.HasOrListParam;
 import ca.uhn.fhir.rest.param.HasParam;
 import ca.uhn.fhir.rest.param.ReferenceOrListParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
+import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import ca.uhn.fhir.util.ExtensionUtil;
 import ca.uhn.fhir.util.HapiExtensions;
 import ca.uhn.fhir.util.Logs;
@@ -121,6 +122,9 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 	@Autowired
 	private IHapiTransactionService myHapiTransactionService;
 
+	@Autowired
+	private ISearchParamRegistry mySearchParamRegistry;
+
 	private IFhirPath myFhirPath;
 
 	@Override
@@ -298,6 +302,10 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 	private LinkedHashSet<JpaPid> getRelatedResourceTypePids(
 			ExportPIDIteratorParameters theParams, RuntimeResourceDefinition theDef) throws IOException {
 		LinkedHashSet<JpaPid> pids = new LinkedHashSet<>();
+		// Check if the patient compartment search parameter is active to enable export of this resource
+		RuntimeSearchParam activeSearchParam =
+				getActivePatientSearchParamForCurrentResourceType(theParams.getResourceType());
+		if (activeSearchParam != null) {
 			// expand the group pid -> list of patients in that group (list of patient pids)
 			Set<JpaPid> expandedMemberResourceIds = expandAllPatientPidsFromGroup(theParams);
 			assert !expandedMemberResourceIds.isEmpty();
@@ -320,6 +328,10 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 					throw new RuntimeException(Msg.code(2346) + "Couldn't close query iterator", ex);
 				}
 			});
+		} else {
+			ourLog.warn("No active patient compartment search parameter(s) for resource type "
+					+ theParams.getResourceType());
+		}
 		return pids;
 	}
 
@@ -600,6 +612,22 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor<JpaPid> {
 		}
 	}
 
+	private RuntimeSearchParam getActivePatientSearchParamForCurrentResourceType(String theResourceType) {
+		String activeSearchParamName = "";
+		String resourceToCheck = theResourceType;
+		if (!PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES.contains(theResourceType)) {
+			activeSearchParamName =
+					getPatientSearchParamForCurrentResourceType(theResourceType).getName();
+		} else if ("Practitioner".equalsIgnoreCase(theResourceType)) {
+			resourceToCheck = "Patient";
+			activeSearchParamName = "general-practitioner";
+		} else if ("Organization".equalsIgnoreCase(theResourceType)) {
+			resourceToCheck = "Patient";
+			activeSearchParamName = "organization";
+		}
+		return mySearchParamRegistry.getActiveSearchParam(resourceToCheck, activeSearchParamName);
+	}
+
 	/**
 	 * Must not be called for resources types listed in PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES
 	 *

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -1400,7 +1400,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 							q.setMaxResults(maxCount);
 						}
 						if (hasDesiredResourceTypes) {
-							q.setParameter("desired_target_resource_types", String.join(", ", desiredResourceTypes));
+							q.setParameter("desired_target_resource_types", desiredResourceTypes);
 						}
 						List<?> results = q.getResultList();
 						for (Object nextRow : results) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.bulk.export.svc;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import ca.uhn.fhir.context.RuntimeSearchParam;
 import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
@@ -23,12 +24,15 @@ import ca.uhn.fhir.mdm.api.MdmMatchResultEnum;
 import ca.uhn.fhir.mdm.dao.IMdmLinkDao;
 import ca.uhn.fhir.mdm.model.MdmPidTuple;
 import ca.uhn.fhir.model.primitive.IdDt;
+import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
 import ca.uhn.fhir.rest.api.server.bulk.BulkExportJobParameters;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.Group;
+import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Patient;
 import org.junit.jupiter.api.Test;
@@ -139,6 +143,9 @@ public class JpaBulkExportProcessorTest {
 	@Mock
 	private MdmExpansionCacheSvc myMdmExpansionCacheSvc;
 
+	@Mock
+	private ISearchParamRegistry mySearchParamRegistry;
+
 	@Spy
 	private IHapiTransactionService myTransactionService = new NonTransactionalHapiTransactionService();
 
@@ -409,6 +416,8 @@ public class JpaBulkExportProcessorTest {
 		ISearchBuilder<JpaPid> observationSearchBuilder = mock(ISearchBuilder.class);
 
 		// when
+		RuntimeSearchParam searchParam = new RuntimeSearchParam(new IdType("1"), "", "", "", "", RestSearchParameterTypeEnum.STRING, Collections.singleton(""), Collections.singleton(""), RuntimeSearchParam.RuntimeSearchParamStatusEnum.ACTIVE, Collections.singleton(""));
+		when(mySearchParamRegistry.getActiveSearchParam(any(), any())).thenReturn(searchParam);
 		// expandAllPatientPidsFromGroup
 		when(myDaoRegistry.getResourceDao(eq("Group")))
 			.thenReturn(groupDao);

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java

@@ -12,12 +12,16 @@ import ca.uhn.fhir.jpa.api.model.BulkExportJobResults;
 import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.RequestTypeEnum;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
 import ca.uhn.fhir.rest.api.server.bulk.BulkExportJobParameters;
 import ca.uhn.fhir.rest.client.apache.ResourceEntity;
+import ca.uhn.fhir.rest.param.StringParam;
+import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.provider.ProviderConstants;
 import ca.uhn.fhir.test.utilities.HttpClientExtension;
@@ -50,6 +54,7 @@ import org.hl7.fhir.r4.model.Practitioner;
 import org.hl7.fhir.r4.model.Provenance;
 import org.hl7.fhir.r4.model.QuestionnaireResponse;
 import org.hl7.fhir.r4.model.Reference;
+import org.hl7.fhir.r4.model.SearchParameter;
 import org.hl7.fhir.r4.model.ServiceRequest;
 import org.hl7.fhir.r4.model.StringType;
 import org.junit.jupiter.api.AfterEach;
@@ -149,6 +154,99 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 		verifyBulkExportResults(options, Collections.singletonList("Patient/PF"), Collections.singletonList("Patient/PM"));
 	}
 
+	@Test
+	public void testGroupBulkExportWithMissingObservationSearchParams() {
+		mySearchParameterDao.update(createDisabledObservationPatientSearchParameter(), mySrd);
+		mySearchParamRegistry.forceRefresh();
+
+		// Create some resources
+		Patient patient = new Patient();
+		patient.setId("PF");
+		patient.setGender(Enumerations.AdministrativeGender.FEMALE);
+		patient.setActive(true);
+		myClient.update().resource(patient).execute();
+
+		patient = new Patient();
+		patient.setId("PM");
+		patient.setGender(Enumerations.AdministrativeGender.MALE);
+		patient.setActive(true);
+		myClient.update().resource(patient).execute();
+
+		Group group = new Group();
+		group.setId("Group/G");
+		group.setActive(true);
+		group.addMember().getEntity().setReference("Patient/PF");
+		group.addMember().getEntity().setReference("Patient/PM");
+		myClient.update().resource(group).execute();
+
+		Observation observation = new Observation();
+		observation.setStatus(Observation.ObservationStatus.AMENDED);
+		observation.setSubject(new Reference("Patient/PF"));
+		String obsId = myClient.create().resource(observation).execute().getId().toUnqualifiedVersionless().getValue();
+
+		// set the export options
+		BulkExportJobParameters options = new BulkExportJobParameters();
+		options.setResourceTypes(Sets.newHashSet("Patient", "Observation"));
+		options.setGroupId("Group/G");
+		options.setExportStyle(BulkExportJobParameters.ExportStyle.GROUP);
+		options.setOutputFormat(Constants.CT_FHIR_NDJSON);
+		verifyBulkExportResults(options, List.of("Patient/PF", "Patient/PM"), Collections.singletonList(obsId));
+	}
+
+	@Test
+	public void testGroupBulkExportWithMissingPatientSearchParams() {
+		mySearchParameterDao.update(createDisabledPatientPractitionerSearchParameter(), mySrd);
+		mySearchParamRegistry.forceRefresh();
+
+		Practitioner practitioner = new Practitioner();
+		practitioner.setActive(true);
+		String practId = myClient.create().resource(practitioner).execute().getId().toUnqualifiedVersionless().getValue();
+
+		// Create some resources
+		Patient patient = new Patient();
+		patient.setId("P1");
+		patient.setActive(true);
+		patient.addGeneralPractitioner().setReference(practId);
+		myClient.update().resource(patient).execute();
+
+		Group group = new Group();
+		group.setId("Group/G");
+		group.setActive(true);
+		group.addMember().getEntity().setReference("Patient/P1");
+		myClient.update().resource(group).execute();
+
+		// set the export options
+		BulkExportJobParameters options = new BulkExportJobParameters();
+		options.setResourceTypes(Sets.newHashSet("Patient", "Practitioner"));
+		options.setGroupId("Group/G");
+		options.setExportStyle(BulkExportJobParameters.ExportStyle.GROUP);
+		options.setOutputFormat(Constants.CT_FHIR_NDJSON);
+		verifyBulkExportResults(options, Collections.singletonList("Patient/P1"), Collections.singletonList(practId));
+	}
+
+	private SearchParameter createDisabledObservationPatientSearchParameter() {
+		SearchParameter observation_patient = new SearchParameter();
+		observation_patient.setId("clinical-patient");
+		observation_patient.addBase("Observation");
+		observation_patient.setStatus(Enumerations.PublicationStatus.RETIRED);
+		observation_patient.setCode("patient");
+		observation_patient.setType(Enumerations.SearchParamType.REFERENCE);
+		observation_patient.addTarget("Patient");
+		observation_patient.setExpression("Observation.subject.where(resolve() is Patient)");
+		return observation_patient;
+	}
+
+	private SearchParameter createDisabledPatientPractitionerSearchParameter() {
+		SearchParameter patient_practitioner = new SearchParameter();
+		patient_practitioner.setId("Patient-general-practitioner");
+		patient_practitioner.addBase("Patient");
+		patient_practitioner.setStatus(Enumerations.PublicationStatus.RETIRED);
+		patient_practitioner.setCode("general-practitioner");
+		patient_practitioner.setType(Enumerations.SearchParamType.REFERENCE);
+		patient_practitioner.addTarget("Practitioner");
+		patient_practitioner.setExpression("Patient.generalPractitioner");
+		return patient_practitioner;
+	}
 
 	@Test
 	public void testGroupBulkExportWithTypeFilter_OnTags_InlineTagMode() {
@@ -446,12 +544,12 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 
 		// set the export options
 		BulkExportJobParameters options = new BulkExportJobParameters();
-		options.setResourceTypes(Sets.newHashSet("Patient", "Encounter"));
+		options.setResourceTypes(Sets.newHashSet("Patient", "Encounter", "Practitioner", "Organization"));
 		options.setGroupId("Group/G1");
 		options.setFilters(new HashSet<>());
 		options.setExportStyle(BulkExportJobParameters.ExportStyle.GROUP);
 		options.setOutputFormat(Constants.CT_FHIR_NDJSON);
-		verifyBulkExportResults(options, List.of("Patient/P1", practId, orgId, encId, encId2, locId), List.of("Patient/P2", orgId2, encId3, locId2));
+		verifyBulkExportResults(options, List.of("Patient/P1", practId, orgId, encId, encId2), List.of("Patient/P2", orgId2, encId3));
 	}
 
 	@Test
@@ -487,7 +585,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 
 		// set the export options
 		BulkExportJobParameters options = new BulkExportJobParameters();
-		options.setResourceTypes(Sets.newHashSet("Patient", "Encounter", "Observation"));
+		options.setResourceTypes(Sets.newHashSet("Patient", "Encounter", "Observation", "Practitioner"));
 		options.setGroupId("Group/G1");
 		options.setFilters(new HashSet<>());
 		options.setExportStyle(BulkExportJobParameters.ExportStyle.GROUP);
@@ -544,7 +642,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 
 		// set the export options
 		BulkExportJobParameters options = new BulkExportJobParameters();
-		options.setResourceTypes(Sets.newHashSet("Patient", "Observation", "Provenance"));
+		options.setResourceTypes(Sets.newHashSet("Patient", "Observation", "Provenance", "Device"));
 		options.setGroupId("Group/G1");
 		options.setFilters(new HashSet<>());
 		options.setExportStyle(BulkExportJobParameters.ExportStyle.GROUP);
@@ -944,10 +1042,7 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test {
 				} catch (IOException e) {
 					fail(e.toString());
 				}
-
 			}
-
-			return jobInstance;
 		}
 
 		for (String containedString : theContainedList) {