Merge pull request #2657 from hapifhir/issue-2655-configurable-expunge-size

Toggleable searchForIds size

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_4_0/2655-configurable-internal-search-size.yaml

@@ -0,0 +1,5 @@
+---
+type: change
+issue: 2655
+title: "Added a new configuration option to DaoConfig, `setInternalSynchronousSearchSize()`, this controls the loadSynchronousUpTo()
+        during internal operations such as delete with expunge, and certain CodeSystem searches."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa/configuration.md

@@ -126,3 +126,7 @@ With this interceptor in place, the following header can be added to individual
 ```http
 X-Retry-On-Version-Conflict: retry; max-retries=100
 ```    
+
+# Controlling Delete with Expunge size
+
+During the delete with expunge operation there is an internal synchronous search which locates all the resources to be deleted. The default maximum size of this search is 10000. This can be configured via the [Internal Synchronous Search Size](/hapi-fhir/apidocs/hapi-fhir-jpaserver-api/ca/uhn/fhir/jpa/api/config/DaoConfig.html#setInternalSynchronousSearchSize(int)) property.

hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java

@@ -89,6 +89,7 @@ public class DaoConfig {
 	/**
 	 * Child Configurations
 	 */
+	private static final Integer DEFAULT_INTERNAL_SYNCHRONOUS_SEARCH_SIZE = 10000;
 
 	private final ModelConfig myModelConfig = new ModelConfig();
 	/**
@@ -155,6 +156,7 @@ public class DaoConfig {
 	private boolean myFilterParameterEnabled = false;
 	private StoreMetaSourceInformationEnum myStoreMetaSourceInformation = StoreMetaSourceInformationEnum.SOURCE_URI_AND_REQUEST_ID;
 	private HistoryCountModeEnum myHistoryCountMode = DEFAULT_HISTORY_COUNT_MODE;
+	private int myInternalSynchronousSearchSize = DEFAULT_INTERNAL_SYNCHRONOUS_SEARCH_SIZE;
 
 	/**
 	 * update setter javadoc if default changes
@@ -2181,6 +2183,30 @@ public class DaoConfig {
 		// ignore
 	}
 
+	/**
+	 * <p>
+	 *    This determines the internal search size that is run synchronously during operations such as:
+	 *    1. Delete with _expunge parameter.
+	 *    2. Searching for Code System IDs by System and Code
+	 * </p>
+	 * @since 5.4.0
+	 */
+	public Integer getInternalSynchronousSearchSize() {
+		return myInternalSynchronousSearchSize;
+	}
+
+	/**
+	 * <p>
+	 *    This determines the internal search size that is run synchronously during operations such as:
+	 *    1. Delete with _expunge parameter.
+	 *    2. Searching for Code System IDs by System and Code
+	 * </p>
+	 * @since 5.4.0
+	 */
+	public void setInternalSynchronousSearchSize(Integer theInternalSynchronousSearchSize) {
+		myInternalSynchronousSearchSize = theInternalSynchronousSearchSize;
+	}
+
 	public enum StoreMetaSourceInformationEnum {
 		NONE(false, false),
 		SOURCE_URI(true, false),

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -1446,7 +1446,7 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	@Override
 	public Set<ResourcePersistentId> searchForIds(SearchParameterMap theParams, RequestDetails theRequest) {
 		return myTransactionService.execute(theRequest, tx -> {
-			theParams.setLoadSynchronousUpTo(10000);
+			theParams.setLoadSynchronousUpTo(myDaoConfig.getInternalSynchronousSearchSize());
 
 			ISearchBuilder builder = mySearchBuilderFactory.newSearchBuilder(this, getResourceName(), getResourceType());
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeService.java

@@ -69,8 +69,6 @@ public class DeleteExpungeService {
 	@Autowired
 	private ResourceTableFKProvider myResourceTableFKProvider;
 	@Autowired
-	private IResourceTableDao myResourceTableDao;
-	@Autowired
 	private IResourceLinkDao myResourceLinkDao;
 	@Autowired
 	private IInterceptorBroadcaster myInterceptorBroadcaster;

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeServiceTest.java

@@ -4,6 +4,8 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
 import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.Organization;
@@ -16,8 +18,12 @@ import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.List;
 
+import static org.hamcrest.Matchers.hasSize;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 class DeleteExpungeServiceTest extends BaseJpaR4Test {
 
@@ -29,6 +35,8 @@ class DeleteExpungeServiceTest extends BaseJpaR4Test {
 		myDaoConfig.setAllowMultipleDelete(true);
 		myDaoConfig.setExpungeEnabled(true);
 		myDaoConfig.setDeleteExpungeEnabled(true);
+		myDaoConfig.setInternalSynchronousSearchSize(new DaoConfig().getInternalSynchronousSearchSize());
+
 	}
 
 	@AfterEach
@@ -57,6 +65,32 @@ class DeleteExpungeServiceTest extends BaseJpaR4Test {
 			assertEquals(e.getMessage(), "DELETE with _expunge=true failed.  Unable to delete " + organizationId.toVersionless() + " because " + patientId.toVersionless() + " refers to it via the path Patient.managingOrganization");
 		}
 	}
+	@Test
+	public void testDeleteExpungeRespectsSynchronousSize() {
+		//Given
+		myDaoConfig.setInternalSynchronousSearchSize(1);
+		Patient patient = new Patient();
+		myPatientDao.create(patient);
+		Patient otherPatient = new Patient();
+		myPatientDao.create(otherPatient);
+
+
+		//When
+		DeleteMethodOutcome deleteMethodOutcome = myPatientDao.deleteByUrl("Patient?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd);
+		IBundleProvider remaining = myPatientDao.search(new SearchParameterMap().setLoadSynchronous(true));
+
+		//Then
+		assertThat(deleteMethodOutcome.getExpungedResourcesCount(), is(equalTo(1L)));
+		assertThat(remaining.size(), is(equalTo(1)));
+
+		//When
+		deleteMethodOutcome = myPatientDao.deleteByUrl("Patient?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd);
+		remaining = myPatientDao.search(new SearchParameterMap().setLoadSynchronous(true));
+
+		//Then
+		assertThat(deleteMethodOutcome.getExpungedResourcesCount(), is(equalTo(1L)));
+		assertThat(remaining.size(), is(equalTo(0)));
+	}
 
 	@Test
 	public void testDeleteExpungeNoThrowExceptionWhenLinkInSearchResults() {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java

@@ -162,6 +162,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
 		myDaoConfig.setEnforceReferentialIntegrityOnDelete(new DaoConfig().isEnforceReferentialIntegrityOnDelete());
 		myDaoConfig.setEnforceReferenceTargetTypes(new DaoConfig().isEnforceReferenceTargetTypes());
 		myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields());
+		myDaoConfig.setInternalSynchronousSearchSize(new DaoConfig().getInternalSynchronousSearchSize());
 		myModelConfig.setNormalizedQuantitySearchLevel(NormalizedQuantitySearchLevel.NORMALIZED_QUANTITY_SEARCH_NOT_SUPPORTED);
 		myDaoConfig.setHistoryCountMode(DaoConfig.DEFAULT_HISTORY_COUNT_MODE);
 	}