Concurrent cascade delete operations leading to confusing errors (#4126)

* Add unit tests and supporting classes from gitlab issue.

* Commit with changes thus far to be reviewed.   Namely, add savepoint capabilities to HapiTransactionService, inject HapiTransactionService into CascadingDeleteInterceptor, add savepoint capabilities to the TransactionDetails, and catch an Exception in BaseHapiFhirResourceDao.delete() then rollback to the checkpoint.

* Fix compile error.

* hapi transaction service test

* hapi transaction service test

* hapi transaction service test

* First draft of RetryTemplate and flush() logic.  Try unit testing with PointcutLatch.

* hapi transaction service test

* Latest changes including moving RetryTemplate, cleaning up unit tests.

* First attempt at correct design.  Fix compile errors.

* Add failed rollback savepoint test.

* fix propogation

* interesting test failure in SafeDeleterTest

* Latest code

* move delete

* test passing

* all tests pass

* yay we're done just cleanup

* First round of cleanup:  Get rid of all exploratory logging, remove unneeded dependencies, move creation of RetryTemplate to SafeDeleter.

* More cleanup.  Add changelog.

* Bump to 6.2.0-PRE13-SNAPSHOT.

* Code review feedback:  Refactor SafeDeleter to a service and bean.  Add comments.

* Code review feedback:  Comments on unit test assertions.

* Code review feedback:  Rename SafeDeleterSvc to ThreadSafeResourceDeleterSvc.  Add javadoc.  Add logger to DelayListener.   Remove useless commented out code.

* Fix unit test failure in CascadingDeleteInterceptorTest.

Co-authored-by: Ken Stevens <ken@smilecdr.com>

hapi-deployable-pom/pom.xml

@@ -4,7 +4,7 @@
     <parent>
 		 <groupId>ca.uhn.hapi.fhir</groupId>
 		 <artifactId>hapi-fhir</artifactId>
-		 <version>6.2.0-PRE12-SNAPSHOT</version>
+		 <version>6.2.0-PRE13-SNAPSHOT</version>
 		 <relativePath>../pom.xml</relativePath>
 	 </parent>
 

hapi-fhir-android/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-batch/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-bom/pom.xml

@@ -3,14 +3,14 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>ca.uhn.hapi.fhir</groupId>
     <artifactId>hapi-fhir-bom</artifactId>
-    <version>6.2.0-PRE12-SNAPSHOT</version>
+    <version>6.2.0-PRE13-SNAPSHOT</version>
     <packaging>pom</packaging>
 	 <name>HAPI FHIR BOM</name>
 
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
     

hapi-fhir-checkstyle/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-api/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-app/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-cli</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/JpaServerDemo.java

@@ -30,6 +30,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
 import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider;
 import ca.uhn.fhir.jpa.config.JpaConfig;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.provider.JpaCapabilityStatementProvider;
 import ca.uhn.fhir.jpa.provider.JpaConformanceProviderDstu2;
@@ -179,7 +180,8 @@ public class JpaServerDemo extends RestfulServer {
 
 		DaoRegistry daoRegistry = myAppCtx.getBean(DaoRegistry.class);
 		IInterceptorBroadcaster interceptorBroadcaster = myAppCtx.getBean(IInterceptorBroadcaster.class);
-		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(ctx, daoRegistry, interceptorBroadcaster);
+		ThreadSafeResourceDeleterSvc threadSafeResourceDeleterSvc = myAppCtx.getBean(ThreadSafeResourceDeleterSvc.class);
+		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(ctx, daoRegistry, interceptorBroadcaster, threadSafeResourceDeleterSvc);
 		getInterceptorService().registerInterceptor(cascadingDeleteInterceptor);
 
 		getInterceptorService().registerInterceptor(new ResponseHighlighterInterceptor());

hapi-fhir-cli/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client-okhttp/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-converter/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-dist/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/3394-cascading-delete-multiple-thread-collision-resource-gone-or-version-conflict.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 3394
+jira: SMILE-3585
+title: "Cascading deletes don't work correctly if multiple threads initiate a delete at the same time.  Either the resource won't be found or there will be a collision on inserting the new version.  This changes fixes the problem by better handling these conditions to either ignore an already deleted resource or to keep retrying in a new inner transaction.."

hapi-fhir-jacoco/pom.xml

@@ -11,7 +11,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jaxrsserver-base/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpa/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-jpaserver-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java

@@ -41,6 +41,7 @@ import ca.uhn.fhir.jpa.dao.index.SearchParamWithInlineReferencesExtractor;
 import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
 import ca.uhn.fhir.jpa.delete.DeleteConflictFinderService;
 import ca.uhn.fhir.jpa.delete.DeleteConflictService;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.graphql.DaoRegistryGraphQLStorageServices;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
@@ -197,8 +198,14 @@ public class JpaConfig {
 
 	@Lazy
 	@Bean
-	public CascadingDeleteInterceptor cascadingDeleteInterceptor(FhirContext theFhirContext, DaoRegistry theDaoRegistry, IInterceptorBroadcaster theInterceptorBroadcaster) {
+	public CascadingDeleteInterceptor cascadingDeleteInterceptor(FhirContext theFhirContext, DaoRegistry theDaoRegistry, IInterceptorBroadcaster theInterceptorBroadcaster, ThreadSafeResourceDeleterSvc threadSafeResourceDeleterSvc) {
-		return new CascadingDeleteInterceptor(theFhirContext, theDaoRegistry, theInterceptorBroadcaster);
+		return new CascadingDeleteInterceptor(theFhirContext, theDaoRegistry, theInterceptorBroadcaster, threadSafeResourceDeleterSvc);
+	}
+
+	@Lazy
+	@Bean
+	public ThreadSafeResourceDeleterSvc safeDeleter(DaoRegistry theDaoRegistry, IInterceptorBroadcaster theInterceptorBroadcaster, HapiTransactionService hapiTransactionService) {
+		return new ThreadSafeResourceDeleterSvc(theDaoRegistry, theInterceptorBroadcaster, hapiTransactionService.getTransactionManager());
 	}
 
 	@Lazy

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/ThreadSafeResourceDeleterSvc.java

@@ -0,0 +1,123 @@
+package ca.uhn.fhir.jpa.delete;
+
+import ca.uhn.fhir.interceptor.api.HookParams;
+import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
+import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
+import ca.uhn.fhir.jpa.api.model.DeleteConflict;
+import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
+import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
+import ca.uhn.fhir.model.primitive.IdDt;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
+import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
+import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
+import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
+import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.retry.backoff.FixedBackOffPolicy;
+import org.springframework.retry.policy.SimpleRetryPolicy;
+import org.springframework.retry.support.RetryTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Used by {@link CascadingDeleteInterceptor} to handle {@link DeleteConflictList}s in a thead-safe way.
+ *
+ * Specifically, this class spawns an inner transaction for each {@link DeleteConflictList}.  This class is meant to handle any potential delete collisions (ex {@link ResourceGoneException} or {@link ResourceVersionConflictException}.  In the former case, we swallow the Exception in the inner transaction then continue.  In the latter case, we retry according to the RETRY_BACKOFF_PERIOD and RETRY_MAX_ATTEMPTS before giving up.
+ */
+public class ThreadSafeResourceDeleterSvc {
+	public static final long RETRY_BACKOFF_PERIOD = 100L;
+	public static final int RETRY_MAX_ATTEMPTS = 4;
+
+	private static final Logger ourLog = LoggerFactory.getLogger(ThreadSafeResourceDeleterSvc.class);
+	private final DaoRegistry myDaoRegistry;
+	private final IInterceptorBroadcaster myInterceptorBroadcaster;
+	private final TransactionTemplate myTxTemplate;
+
+	private final RetryTemplate myRetryTemplate = getRetryTemplate();
+
+	public ThreadSafeResourceDeleterSvc(DaoRegistry theDaoRegistry, IInterceptorBroadcaster theInterceptorBroadcaster, PlatformTransactionManager thePlatformTransactionManager) {
+		myDaoRegistry = theDaoRegistry;
+		myInterceptorBroadcaster = theInterceptorBroadcaster;
+		myTxTemplate = new TransactionTemplate(thePlatformTransactionManager);
+		myTxTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
+	}
+
+	/**
+	 * @return number of resources that were successfully deleted
+	 */
+	public Integer delete(RequestDetails theRequest, DeleteConflictList theConflictList, TransactionDetails theTransactionDetails) {
+		Integer retVal = 0;
+
+		List<String> cascadeDeleteIdCache = CascadingDeleteInterceptor.getCascadedDeletesList(theRequest, true);
+		for (DeleteConflict next : theConflictList) {
+			IdDt nextSource = next.getSourceId();
+			String nextSourceId = nextSource.toUnqualifiedVersionless().getValue();
+
+			if (!cascadeDeleteIdCache.contains(nextSourceId)) {
+				cascadeDeleteIdCache.add(nextSourceId);
+				retVal += handleNextSource(theRequest, theConflictList, theTransactionDetails, next, nextSource, nextSourceId);
+			}
+		}
+
+		return retVal;
+	}
+
+	/**
+	 * @return number of resources that were successfully deleted
+	 */
+	private Integer handleNextSource(RequestDetails theRequest, DeleteConflictList theConflictList, TransactionDetails theTransactionDetails, DeleteConflict next, IdDt nextSource, String nextSourceId) {
+		IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(nextSource.getResourceType());
+
+		// We will retry deletes on any occurrence of ResourceVersionConflictException up to RETRY_MAX_ATTEMPTS
+		return myRetryTemplate.execute(retryContext -> {
+			try {
+				if (retryContext.getRetryCount() > 0) {
+					ourLog.info("Retrying delete of {} - Attempt #{}", nextSourceId, retryContext.getRetryCount());
+				}
+				myTxTemplate.execute(s -> doDelete(theRequest, theConflictList, theTransactionDetails, nextSource, dao));
+				return 1;
+			} catch (ResourceGoneException exception) {
+				ourLog.info("{} is already deleted.  Skipping cascade delete of this resource", nextSourceId);
+			}
+			return 0;
+		});
+	}
+
+	private DaoMethodOutcome doDelete(RequestDetails theRequest, DeleteConflictList
+		theConflictList, TransactionDetails theTransactionDetails, IdDt nextSource, IFhirResourceDao<?> dao) {
+		// Interceptor call: STORAGE_CASCADE_DELETE
+
+		// Remove the version so we grab the latest version to delete
+		IBaseResource resource = dao.read(nextSource.toVersionless(), theRequest);
+		HookParams params = new HookParams()
+			.add(RequestDetails.class, theRequest)
+			.addIfMatchesType(ServletRequestDetails.class, theRequest)
+			.add(DeleteConflictList.class, theConflictList)
+			.add(IBaseResource.class, resource);
+		CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_CASCADE_DELETE, params);
+
+		return dao.delete(resource.getIdElement(), theConflictList, theRequest, theTransactionDetails);
+	}
+
+	private static RetryTemplate getRetryTemplate() {
+		final RetryTemplate retryTemplate = new RetryTemplate();
+
+		final FixedBackOffPolicy fixedBackOffPolicy = new FixedBackOffPolicy();
+		fixedBackOffPolicy.setBackOffPeriod(RETRY_BACKOFF_PERIOD);
+		retryTemplate.setBackOffPolicy(fixedBackOffPolicy);
+
+		final SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(RETRY_MAX_ATTEMPTS, Collections.singletonMap(ResourceVersionConflictException.class, true));
+		retryTemplate.setRetryPolicy(retryPolicy);
+
+		return retryTemplate;
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/interceptor/CascadingDeleteInterceptor.java

@@ -22,23 +22,18 @@ package ca.uhn.fhir.jpa.interceptor;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.interceptor.api.Hook;
-import ca.uhn.fhir.interceptor.api.HookParams;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
 import ca.uhn.fhir.interceptor.api.Interceptor;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
-import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
-import ca.uhn.fhir.jpa.api.model.DeleteConflict;
 import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
 import ca.uhn.fhir.jpa.delete.DeleteConflictOutcome;
-import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
-import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
-import ca.uhn.fhir.model.primitive.IdDt;
 import ca.uhn.fhir.rest.api.DeleteCascadeModeEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.ResponseDetails;
+import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import ca.uhn.fhir.rest.server.RestfulServerUtils;
-import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.util.OperationOutcomeUtil;
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
@@ -85,20 +80,23 @@ public class CascadingDeleteInterceptor {
 	private final DaoRegistry myDaoRegistry;
 	private final IInterceptorBroadcaster myInterceptorBroadcaster;
 	private final FhirContext myFhirContext;
+	private final ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
 
 	/**
 	 * Constructor
 	 *
 	 * @param theDaoRegistry The DAO registry (must not be null)
 	 */
-	public CascadingDeleteInterceptor(@Nonnull FhirContext theFhirContext, @Nonnull DaoRegistry theDaoRegistry, @Nonnull IInterceptorBroadcaster theInterceptorBroadcaster) {
+	public CascadingDeleteInterceptor(@Nonnull FhirContext theFhirContext, @Nonnull DaoRegistry theDaoRegistry, @Nonnull IInterceptorBroadcaster theInterceptorBroadcaster, @Nonnull ThreadSafeResourceDeleterSvc theThreadSafeResourceDeleterSvc) {
 		Validate.notNull(theDaoRegistry, "theDaoRegistry must not be null");
 		Validate.notNull(theInterceptorBroadcaster, "theInterceptorBroadcaster must not be null");
 		Validate.notNull(theFhirContext, "theFhirContext must not be null");
+		Validate.notNull(theThreadSafeResourceDeleterSvc, "theSafeDeleter must not be null");
 
 		myDaoRegistry = theDaoRegistry;
 		myInterceptorBroadcaster = theInterceptorBroadcaster;
 		myFhirContext = theFhirContext;
+		myThreadSafeResourceDeleterSvc = theThreadSafeResourceDeleterSvc;
 	}
 
 	@Hook(value = Pointcut.STORAGE_PRESTORAGE_DELETE_CONFLICTS, order = CASCADING_DELETE_INTERCEPTOR_ORDER)
@@ -118,36 +116,12 @@ public class CascadingDeleteInterceptor {
 			return null;
 		}
 
-		List<String> cascadedDeletes = getCascadedDeletesMap(theRequest, true);
+		myThreadSafeResourceDeleterSvc.delete(theRequest, theConflictList, theTransactionDetails);
-		for (DeleteConflict next : theConflictList) {
-			IdDt nextSource = next.getSourceId();
-			String nextSourceId = nextSource.toUnqualifiedVersionless().getValue();
-
-			if (!cascadedDeletes.contains(nextSourceId)) {
-				cascadedDeletes.add(nextSourceId);
-
-				IFhirResourceDao dao = myDaoRegistry.getResourceDao(nextSource.getResourceType());
-
-				// Interceptor call: STORAGE_CASCADE_DELETE
-				IBaseResource resource = dao.read(nextSource, theRequest);
-				HookParams params = new HookParams()
-					.add(RequestDetails.class, theRequest)
-					.addIfMatchesType(ServletRequestDetails.class, theRequest)
-					.add(DeleteConflictList.class, theConflictList)
-					.add(IBaseResource.class, resource);
-				CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_CASCADE_DELETE, params);
-
-				// Actually perform the delete
-				ourLog.info("Have delete conflict {} - Cascading delete", next);
-				dao.delete(nextSource, theConflictList, theRequest, theTransactionDetails);
-			}
-		}
 
 		return new DeleteConflictOutcome().setShouldRetryCount(MAX_RETRY_ATTEMPTS);
 	}
 
-	@SuppressWarnings("unchecked")
+	public static List<String> getCascadedDeletesList(RequestDetails theRequest, boolean theCreate) {
-	private List<String> getCascadedDeletesMap(RequestDetails theRequest, boolean theCreate) {
 		List<String> retVal = (List<String>) theRequest.getUserData().get(CASCADED_DELETES_KEY);
 		if (retVal == null && theCreate) {
 			retVal = new ArrayList<>();
@@ -178,7 +152,7 @@ public class CascadingDeleteInterceptor {
 		if (theRequestDetails != null) {
 
 			// Successful delete list
-			List<String> deleteList = getCascadedDeletesMap(theRequestDetails, false);
+			List<String> deleteList = getCascadedDeletesList(theRequestDetails, false);
 			if (deleteList != null) {
 				if (theResponseDetails.getResponseCode() == 200) {
 					if (theResponse instanceof IBaseOperationOutcome) {
@@ -206,6 +180,4 @@ public class CascadingDeleteInterceptor {
 	protected DeleteCascadeModeEnum shouldCascade(@Nullable RequestDetails theRequest) {
 		return RestfulServerUtils.extractDeleteCascadeParameter(theRequest);
 	}
-
-
 }

hapi-fhir-jpaserver-cql/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-elastic-test-utilities/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-mdm/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-model/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-searchparam/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-subscription/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu2/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu3/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirSystemDaoDstu3Test.java

@@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
 import ca.uhn.fhir.jpa.dao.GZipUtil;
 import ca.uhn.fhir.jpa.dao.r4.FhirSystemDaoR4;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.model.entity.ResourceTag;
 import ca.uhn.fhir.jpa.model.entity.TagTypeEnum;
@@ -110,6 +111,8 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
 	private DaoRegistry myDaoRegistry;
 	@Autowired
 	private IInterceptorService myInterceptorBroadcaster;
+	@Autowired
+	private ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
 
 	@AfterEach
 	public void after() {
@@ -1759,7 +1762,7 @@ public class FhirSystemDaoDstu3Test extends BaseJpaDstu3SystemTest {
 		params.put(Constants.PARAMETER_CASCADE_DELETE, new String[]{Constants.CASCADE_DELETE});
 		mySrd.setParameters(params);
 
-		CascadingDeleteInterceptor deleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorBroadcaster);
+		CascadingDeleteInterceptor deleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorBroadcaster, myThreadSafeResourceDeleterSvc);
 		myInterceptorBroadcaster.registerInterceptor(deleteInterceptor);
 
 

hapi-fhir-jpaserver-test-r4/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/delete/ThreadSafeResourceDeleterSvcTest.java

@@ -0,0 +1,256 @@
+package ca.uhn.fhir.jpa.delete;
+
+import ca.uhn.fhir.interceptor.api.Hook;
+import ca.uhn.fhir.interceptor.api.HookParams;
+import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
+import ca.uhn.fhir.interceptor.api.IInterceptorService;
+import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.api.model.DeleteConflict;
+import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
+import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
+import ca.uhn.fhir.model.primitive.IdDt;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
+import ca.uhn.test.concurrency.IPointcutLatch;
+import ca.uhn.test.concurrency.PointcutLatch;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.HumanName;
+import org.hl7.fhir.r4.model.Patient;
+import javax.annotation.Nullable;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ThreadSafeResourceDeleterSvcTest extends BaseJpaR4Test {
+	private static final String PATIENT1_ID = "a1";
+	private static final String PATIENT2_ID = "a2";
+
+	private final PointcutLatch myPointcutLatch = new PointcutLatch(Pointcut.STORAGE_CASCADE_DELETE);
+
+	private final MyCascadeDeleteInterceptor myCascadeDeleteInterceptor = new MyCascadeDeleteInterceptor();
+
+	private ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
+	@Autowired
+	IInterceptorBroadcaster myIdInterceptorBroadcaster;
+
+	@Autowired
+	HapiTransactionService myHapiTransactionService;
+	@Autowired
+	PlatformTransactionManager myPlatformTransactionManager;
+
+	@Autowired
+	private IInterceptorService myInterceptorService;
+
+	private final TransactionDetails myTransactionDetails = new TransactionDetails();
+
+
+	@BeforeEach
+	void beforeEach() {
+		myThreadSafeResourceDeleterSvc = new ThreadSafeResourceDeleterSvc(myDaoRegistry, myIdInterceptorBroadcaster, myPlatformTransactionManager);
+	}
+
+	@AfterEach
+	void tearDown() {
+		myInterceptorService.unregisterInterceptor(myPointcutLatch);
+		myInterceptorService.unregisterInterceptor(myCascadeDeleteInterceptor);
+		myCascadeDeleteInterceptor.clear();
+	}
+
+	@Test
+	void delete_nothing() {
+		DeleteConflictList conflictList = new DeleteConflictList();
+
+		myThreadSafeResourceDeleterSvc.delete(mySrd, conflictList, myTransactionDetails);
+	}
+
+	@Test
+	void delete_delete_two() {
+		DeleteConflictList conflictList = new DeleteConflictList();
+		IIdType orgId = createOrganization();
+		IIdType patient1Id = createPatientWithVersion(withId(PATIENT1_ID));
+		IIdType patient2Id = createPatientWithVersion(withId(PATIENT2_ID));
+
+		conflictList.add(buildDeleteConflict(patient1Id, orgId));
+		conflictList.add(buildDeleteConflict(patient2Id, orgId));
+
+		assertEquals(2, countPatients());
+		myHapiTransactionService.execute(mySrd, myTransactionDetails, status -> myThreadSafeResourceDeleterSvc.delete(mySrd, conflictList, myTransactionDetails));
+
+		assertEquals(0, countPatients());
+		assertEquals(1, countOrganizations());
+	}
+
+	@Test
+	void delete_delete_retryTest() throws ExecutionException, InterruptedException {
+		myInterceptorService.registerInterceptor(myCascadeDeleteInterceptor);
+
+		DeleteConflictList conflictList = new DeleteConflictList();
+		IIdType orgId = createOrganization();
+		IIdType patient1Id = createPatientWithVersion(withId(PATIENT1_ID));
+		IIdType patient2Id = createPatientWithVersion(withId(PATIENT2_ID));
+
+		conflictList.add(buildDeleteConflict(patient1Id, orgId));
+		conflictList.add(buildDeleteConflict(patient2Id, orgId));
+
+		assertEquals(2, countPatients());
+		final ExecutorService executorService = Executors.newSingleThreadExecutor();
+
+		myCascadeDeleteInterceptor.setExpectedCount(1);
+		ourLog.info("Start background delete");
+		final Future<Integer> future = executorService.submit(() ->
+			myHapiTransactionService.execute(mySrd, myTransactionDetails, status -> myThreadSafeResourceDeleterSvc.delete(mySrd, conflictList, myTransactionDetails)));
+
+		// We are paused before deleting the first patient.
+		myCascadeDeleteInterceptor.awaitExpected();
+
+		//   Let's delete the second patient from under its nose.
+		ourLog.info("delete patient 2");
+		myPatientDao.delete(patient2Id);
+
+		// Unpause and delete the first patient
+		myCascadeDeleteInterceptor.release("first");
+
+		// future.get() returns total number of resources deleted, which in this case is 1
+		assertEquals(1, future.get());
+
+		assertEquals(0, countPatients());
+		assertEquals(1, countOrganizations());
+	}
+
+	@Test
+	void delete_update_retryTest() throws ExecutionException, InterruptedException {
+		myInterceptorService.registerInterceptor(myCascadeDeleteInterceptor);
+
+		DeleteConflictList conflictList = new DeleteConflictList();
+		IIdType orgId = createOrganization();
+		IIdType patient1Id = createPatientWithVersion(withId(PATIENT1_ID));
+		IIdType patient2Id = createPatientWithVersion(withId(PATIENT2_ID));
+
+		conflictList.add(buildDeleteConflict(patient1Id, orgId));
+		conflictList.add(buildDeleteConflict(patient2Id, orgId));
+
+		assertEquals(2, countPatients());
+		final ExecutorService executorService = Executors.newSingleThreadExecutor();
+
+		myCascadeDeleteInterceptor.setExpectedCount(1);
+		final Future<Integer> future = executorService.submit(() ->
+			myHapiTransactionService.execute(mySrd, myTransactionDetails, status -> myThreadSafeResourceDeleterSvc.delete(mySrd, conflictList, myTransactionDetails)));
+
+		// Patient 1 We are paused after reading the version but before deleting the first patient.
+		myCascadeDeleteInterceptor.awaitExpected();
+
+
+		// Unpause and delete the first patient
+		myCascadeDeleteInterceptor.setExpectedCount(1);
+		myCascadeDeleteInterceptor.release("first");
+
+		// Patient 2 We are paused after reading the version but before deleting the second patient.
+		myCascadeDeleteInterceptor.awaitExpected();
+
+		updatePatient(patient2Id);
+
+		// Unpause and fail to delete the second patient
+		myCascadeDeleteInterceptor.setExpectedCount(1);
+		myCascadeDeleteInterceptor.release("second");
+
+		// Red Green: If you delete the updatePatient above, it will timeout here
+		myCascadeDeleteInterceptor.awaitExpected();
+
+		// Unpause and succeed in deleting the second patient because we will get the correct version now
+		myCascadeDeleteInterceptor.release("third");
+
+		// future.get() returns total number of resources deleted, which in this case is 2
+		assertEquals(2, future.get());
+
+		assertEquals(0, countPatients());
+		assertEquals(1, countOrganizations());
+	}
+
+	private void updatePatient(IIdType patient2Id) {
+		//   Let's delete the second patient from under its nose.
+		final Patient patient2 = myPatientDao.read(patient2Id);
+		final HumanName familyName = new HumanName();
+		familyName.setFamily("Doo");
+		patient2.getName().add(familyName);
+
+		ourLog.info("about to update");
+		myPatientDao.update(patient2);
+		ourLog.info("update complete");
+	}
+
+	@Nullable
+	private Integer countPatients() {
+		SearchParameterMap map = SearchParameterMap.newSynchronous();
+		return myPatientDao.search(map).size();
+	}
+
+	@Nullable
+	private Integer countOrganizations() {
+		SearchParameterMap map = SearchParameterMap.newSynchronous();
+		return myOrganizationDao.search(map).size();
+	}
+
+	private IIdType createPatientWithVersion(Consumer<IBaseResource> theWithId) {
+		Patient patient = new Patient();
+		patient.addName(new HumanName().setFamily("FAMILY"));
+		theWithId.accept(patient);
+		return myPatientDao.update(patient, mySrd).getId();
+	}
+
+	private DeleteConflict buildDeleteConflict(IIdType thePatient1Id, IIdType theOrgId) {
+		return new DeleteConflict(new IdDt(thePatient1Id), "managingOrganization", new IdDt(theOrgId));
+	}
+
+	private static class MyCascadeDeleteInterceptor implements IPointcutLatch {
+		private final PointcutLatch myCalledLatch = new PointcutLatch("Called");
+		private final PointcutLatch myWaitLatch = new PointcutLatch("Wait");
+
+		MyCascadeDeleteInterceptor() {
+			myWaitLatch.setExpectedCount(1);
+		}
+
+		@Hook(Pointcut.STORAGE_CASCADE_DELETE)
+		public void cascadeDelete(RequestDetails theRequestDetails, DeleteConflictList theConflictList, IBaseResource theResource) throws InterruptedException {
+			myCalledLatch.call(theResource);
+			ourLog.info("Waiting to proceed with delete");
+			myWaitLatch.awaitExpected();
+			ourLog.info("Cascade Delete proceeding: {}", myWaitLatch.getLatchInvocationParameter());
+			myWaitLatch.setExpectedCount(1);
+		}
+
+		void release(String theMessage) {
+			ourLog.info("Releasing {}", theMessage);
+			myWaitLatch.call(theMessage);
+		}
+
+		@Override
+		public void clear() {
+			myCalledLatch.clear();
+			myWaitLatch.clear();
+		}
+
+		@Override
+		public void setExpectedCount(int count) {
+			myCalledLatch.setExpectedCount(count);
+		}
+
+		@Override
+		public List<HookParams> awaitExpected() throws InterruptedException {
+			return myCalledLatch.awaitExpected();
+		}
+	}
+}

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/CascadingDeleteInterceptorMultiThreadTest.java

@@ -0,0 +1,482 @@
+package ca.uhn.fhir.jpa.interceptor;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
+import ca.uhn.fhir.jpa.api.model.ExpungeOptions;
+import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
+import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper;
+import ca.uhn.fhir.jpa.model.entity.ModelConfig;
+import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
+import ca.uhn.fhir.jpa.searchparam.registry.SearchParamRegistryImpl;
+import ca.uhn.fhir.jpa.test.config.DelayListener;
+import ca.uhn.fhir.jpa.test.config.TestR4WithDelayConfig;
+import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
+import ca.uhn.fhir.parser.StrictErrorHandler;
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.api.EncodingEnum;
+import ca.uhn.fhir.rest.client.api.IGenericClient;
+import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
+import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
+import ca.uhn.fhir.rest.server.RestfulServer;
+import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
+import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory;
+import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
+import ca.uhn.fhir.test.utilities.JettyUtil;
+import com.google.common.base.Charsets;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.Encounter;
+import org.hl7.fhir.r4.model.Meta;
+import org.hl7.fhir.r4.model.Organization;
+import org.hl7.fhir.r4.model.Practitioner;
+import org.hl7.fhir.r4.model.Reference;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.ApplicationContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.support.GenericWebApplicationContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+@ExtendWith(SpringExtension.class)
+@ContextConfiguration(classes = {TestR4WithDelayConfig.class})
+public class CascadingDeleteInterceptorMultiThreadTest {
+
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(CascadingDeleteInterceptorMultiThreadTest.class);
+
+	@Autowired
+	private CascadingDeleteInterceptor myDeleteInterceptor;
+	@Autowired
+	@Qualifier("myResourceProvidersR4")
+	protected ResourceProviderFactory myResourceProviders;
+	@Autowired
+	protected ApplicationContext myAppCtx;
+	@Autowired
+	protected ModelConfig myModelConfig;
+	@Autowired
+	protected FhirContext myFhirContext;
+	@Autowired
+	protected DaoConfig myDaoConfig;
+	@Autowired
+	@Qualifier("mySystemDaoR4")
+	protected IFhirSystemDao<Bundle, Meta> mySystemDao;
+	@Autowired
+	protected IResourceReindexingSvc myResourceReindexingSvc;
+	@Autowired
+	protected ISearchCoordinatorSvc mySearchCoordinatorSvc;
+	@Autowired
+	protected SearchParamRegistryImpl mySearchParamRegistry;
+	@Autowired
+	private IBulkDataExportJobSchedulingHelper myBulkDataScheduleHelper;
+	@Autowired
+	DelayListener myDelayListener;
+	@Autowired
+	protected CircularQueueCaptureQueriesListener myCaptureQueriesListener;
+
+
+	private static Server ourServer;
+	private static RestfulServer ourRestServer;
+	private static String ourServerBase;
+	private IIdType myOrganizationId;
+	private IIdType myPractitionerId;
+	private IGenericClient myClient;
+	private CloseableHttpClient myHttpClient1;
+	private CloseableHttpClient myHttpClient2;
+
+	@BeforeEach
+	public void before() throws Exception {
+		myFhirContext.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+		myFhirContext.getRestfulClientFactory().setSocketTimeout(1200 * 1000);
+		myFhirContext.setParserErrorHandler(new StrictErrorHandler());
+
+		if (ourServer == null) {
+			ourRestServer = new RestfulServer(myFhirContext);
+			ourRestServer.registerProviders(myResourceProviders.createProviders());
+			ourRestServer.setDefaultResponseEncoding(EncodingEnum.XML);
+
+			Server server = new Server(0);
+
+			ServletContextHandler proxyHandler = new ServletContextHandler();
+			proxyHandler.setContextPath("/");
+
+			ServletHolder servletHolder = new ServletHolder();
+			servletHolder.setServlet(ourRestServer);
+			proxyHandler.addServlet(servletHolder, "/fhir/context/*");
+
+			GenericWebApplicationContext ourWebApplicationContext = new GenericWebApplicationContext();
+			ourWebApplicationContext.setParent(myAppCtx);
+			ourWebApplicationContext.refresh();
+			proxyHandler.getServletContext().setAttribute(WebApplicationContext.ROOT_WEB_APPLICATION_CONTEXT_ATTRIBUTE, ourWebApplicationContext);
+
+			server.setHandler(proxyHandler);
+			JettyUtil.startServer(server);
+			int port = JettyUtil.getPortForStartedServer(server);
+			ourServerBase = "http://localhost:" + port + "/fhir/context";
+
+			myFhirContext.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+			myFhirContext.getRestfulClientFactory().setSocketTimeout(400000);
+
+			ourServer = server;
+		}
+
+		myClient = myFhirContext.newRestfulGenericClient(ourServerBase);
+		myClient.registerInterceptor(new LoggingInterceptor());
+
+		myHttpClient1 = getHttpClient();
+		myHttpClient2 = getHttpClient();
+	}
+
+	@AfterEach
+	public void afterTest() throws IOException {
+		purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);
+		if (myCaptureQueriesListener != null) {
+			myCaptureQueriesListener.clear();
+		}
+		myDelayListener.reset();
+		myHttpClient1.close();
+		myHttpClient2.close();
+	}
+
+	protected static void purgeDatabase(DaoConfig theDaoConfig, IFhirSystemDao<?, ?> theSystemDao, IResourceReindexingSvc theResourceReindexingSvc, ISearchCoordinatorSvc theSearchCoordinatorSvc, ISearchParamRegistry theSearchParamRegistry, IBulkDataExportJobSchedulingHelper theBulkDataJobActivator) {
+		theSearchCoordinatorSvc.cancelAllActiveSearches();
+		theResourceReindexingSvc.cancelAndPurgeAllJobs();
+		theBulkDataJobActivator.cancelAndPurgeAllJobs();
+
+		boolean expungeEnabled = theDaoConfig.isExpungeEnabled();
+		boolean multiDeleteEnabled = theDaoConfig.isAllowMultipleDelete();
+		theDaoConfig.setExpungeEnabled(true);
+		theDaoConfig.setAllowMultipleDelete(true);
+
+		for (int count = 0; ; count++) {
+			try {
+				theSystemDao.expunge(new ExpungeOptions().setExpungeEverything(true), null);
+				break;
+			} catch (Exception e) {
+				if (count >= 3) {
+					ourLog.error("Failed during expunge", e);
+					fail(e.toString());
+				} else {
+					try {
+						Thread.sleep(1000);
+					} catch (InterruptedException e2) {
+						fail(e2.toString());
+					}
+				}
+			}
+		}
+		theDaoConfig.setExpungeEnabled(expungeEnabled);
+		theDaoConfig.setAllowMultipleDelete(multiDeleteEnabled);
+
+		theSearchParamRegistry.forceRefresh();
+	}
+
+
+	@AfterAll
+	public static void afterClassClearContextBaseResourceProviderR4Test() throws Exception {
+		JettyUtil.closeServer(ourServer);
+		ourServer = null;
+	}
+
+	public void createResources() {
+		Practitioner prac = new Practitioner();
+		prac.setActive(true);
+		prac.setId("test-prac");
+		myPractitionerId = myClient.update().resource(prac).withId("Practitioner/test-prac").execute().getId().toUnqualifiedVersionless();
+
+		Organization org = new Organization();
+		org.setActive(true);
+		org.setId("test-org");
+		myOrganizationId = myClient.update().resource(org).withId("Organization/test-org").execute().getId().toUnqualifiedVersionless();
+
+		Encounter enc = new Encounter();
+		enc.addParticipant().setIndividual(new Reference(myPractitionerId));
+		enc.setServiceProvider(new Reference(myOrganizationId));
+		enc.setId("test-enc-1");
+		myClient.update().resource(enc).withId("Encounter/test-enc-1").execute();
+
+		enc = new Encounter();
+		enc.addParticipant().setIndividual(new Reference(myPractitionerId));
+		enc.setServiceProvider(new Reference(myOrganizationId));
+		enc.setId("test-enc-2");
+		myClient.update().resource(enc).withId("Encounter/test-enc-2").execute();
+
+		enc = new Encounter();
+		enc.addParticipant().setIndividual(new Reference(myPractitionerId));
+		enc.setServiceProvider(new Reference(myOrganizationId));
+		enc.setId("test-enc-3");
+		myClient.update().resource(enc).withId("Encounter/test-enc-3").execute();
+
+		enc = new Encounter();
+		enc.addParticipant().setIndividual(new Reference(myPractitionerId));
+		enc.setServiceProvider(new Reference(myOrganizationId));
+		enc.setId("test-enc-4");
+		myClient.update().resource(enc).withId("Encounter/test-enc-4").execute();
+
+	}
+
+	@Test
+	public void testDeleteCascadingConcurrentThreadsWithOneDelayed() {
+		myDelayListener.enable();
+
+		myModelConfig.setRespectVersionsForSearchIncludes(false);
+		createResources();
+
+		ourRestServer.getInterceptorService().registerInterceptor(myDeleteInterceptor);
+
+		ExecutorService executor = Executors.newFixedThreadPool(2);
+		Callable<Boolean> job1 = () -> {
+			try {
+				return deleteOrganization(myHttpClient1);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+		Callable<Boolean> job2 = () -> {
+			try {
+				return deletePractitioner(myHttpClient2);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+
+		try {
+			List<Future<Boolean>> futures = new ArrayList<>();
+			futures.add(executor.submit(job1));
+			futures.add(executor.submit(job2));
+			myCaptureQueriesListener.logAllQueriesForCurrentThread();
+			List<Boolean> results = new ArrayList<>();
+			for (Future<Boolean> next : futures) {
+				results.add(next.get());
+			}
+			for (Boolean next : results) {
+				assert(next);
+			}
+		} catch (ExecutionException | InterruptedException theE) {
+			theE.printStackTrace();
+		} finally {
+			executor.shutdown();
+		}
+
+	}
+
+	@Test
+	public void testDeleteCascadingConcurrentThreads() {
+		myModelConfig.setRespectVersionsForSearchIncludes(false);
+		createResources();
+
+		ourRestServer.getInterceptorService().registerInterceptor(myDeleteInterceptor);
+
+		ExecutorService executor = Executors.newFixedThreadPool(2);
+		Callable<Boolean> job1 = () -> {
+			try {
+				return deleteOrganization(myHttpClient1);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+		Callable<Boolean> job2 = () -> {
+			try {
+				return deletePractitioner(myHttpClient2);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+
+		try {
+			List<Future<Boolean>> futures = new ArrayList<>();
+			futures.add(executor.submit(job1));
+			futures.add(executor.submit(job2));
+			List<Boolean> results = new ArrayList<>();
+			for (Future<Boolean> next : futures) {
+				results.add(next.get());
+			}
+			for (Boolean next : results) {
+				assert(next);
+			}
+		} catch (ExecutionException | InterruptedException theE) {
+			theE.printStackTrace();
+		} finally {
+			executor.shutdown();
+		}
+
+	}
+
+	@Test
+	public void testDeleteCascadingSequentialThreads() {
+		myModelConfig.setRespectVersionsForSearchIncludes(false);
+		createResources();
+
+		ourRestServer.getInterceptorService().registerInterceptor(myDeleteInterceptor);
+
+		ExecutorService executor = Executors.newFixedThreadPool(2);
+		Callable<Boolean> job1 = () -> {
+			try {
+				return deleteOrganization(myHttpClient1);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+		Callable<Boolean> job2 = () -> {
+			try {
+				return deletePractitioner(myHttpClient2);
+			} catch (IOException theE) {
+				theE.printStackTrace();
+			}
+			return false;
+		};
+
+		try {
+			Future<Boolean> future1 = executor.submit(job1);
+			// 100ms seems to be too short
+			Thread.sleep(300L);
+			Future<Boolean> future2 = executor.submit(job2);
+			List<Boolean> results = new ArrayList<>();
+			results.add(future1.get());
+			results.add(future2.get());
+			for (Boolean next : results) {
+				assert(next);
+			}
+		} catch (ExecutionException | InterruptedException theE) {
+			theE.printStackTrace();
+		} finally {
+			executor.shutdown();
+		}
+	}
+
+	@Test
+	public void testDeleteCascadingSingleThread() {
+		myModelConfig.setRespectVersionsForSearchIncludes(false);
+		createResources();
+
+		ourRestServer.getInterceptorService().registerInterceptor(myDeleteInterceptor);
+
+		boolean deleteOrganizationSucceeded = false;
+		boolean deletePractitionerSucceeded = false;
+		try {
+			deleteOrganizationSucceeded = deleteOrganization(myHttpClient1);
+			deletePractitionerSucceeded = deletePractitioner(myHttpClient2);
+		} catch (IOException theE) {
+			theE.printStackTrace();
+		}
+		assert(deleteOrganizationSucceeded && deletePractitionerSucceeded);
+
+	}
+
+	private CloseableHttpClient getHttpClient() {
+		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+		connectionManager.setMaxTotal(10);
+		connectionManager.setDefaultMaxPerRoute(10);
+		HttpClientBuilder builder = HttpClientBuilder.create();
+		builder.setConnectionManager(connectionManager);
+		builder.setMaxConnPerRoute(99);
+
+		return builder.build();
+
+	}
+
+	private boolean deletePractitioner(CloseableHttpClient theCloseableHttpClient) throws IOException {
+		ourLog.info("Starting deletePractitioner");
+		HttpDelete delete = new HttpDelete(ourServerBase + "/" + myPractitionerId.getValue() + "?" + Constants.PARAMETER_CASCADE_DELETE + "=" + Constants.CASCADE_DELETE + "&_pretty=true");
+		delete.addHeader(Constants.HEADER_ACCEPT, Constants.CT_FHIR_JSON_NEW);
+		try (CloseableHttpResponse response = theCloseableHttpClient.execute(delete)) {
+			if (response.getStatusLine().getStatusCode() != 200) {
+				ourLog.error("Unexpected status on practitioner delete = " + response.getStatusLine().getStatusCode());
+				return false;
+			}
+			String deleteResponse = IOUtils.toString(response.getEntity().getContent(), Charsets.UTF_8);
+			ourLog.info("Response: {}", deleteResponse);
+			if (!deleteResponse.contains("Cascaded delete to ") && !deleteResponse.contains("Successfully deleted 1 resource(s)")) {
+				ourLog.error("Unexpected response on practitioner delete = " + deleteResponse);
+				return false;
+			}
+		}
+
+		ourLog.info("Delete of Practitioner completed");
+
+		try {
+			ourLog.info("Reading {}", myPractitionerId);
+			myClient.read().resource(Practitioner.class).withId(myPractitionerId).execute();
+			fail();
+		} catch (ResourceGoneException ignored) {
+			ourLog.info("Practitioner resource gone as expected");
+		}
+		try {
+			ourLog.info("Searching for encounters after deleting Practitioner");
+			myClient.read().resource(Encounter.class).withId("Encounter/test-enc-3").execute();
+			fail();
+		} catch (ResourceGoneException ignored) {
+			ourLog.info("Encounter resource gone as expected after Practitioner deleted");
+		}
+		return true;
+	}
+
+	private boolean deleteOrganization(CloseableHttpClient theCloseableHttpClient) throws IOException {
+		ourLog.info("Starting deleteOrganization");
+		HttpDelete delete = new HttpDelete(ourServerBase + "/" + myOrganizationId.getValue() + "?" + Constants.PARAMETER_CASCADE_DELETE + "=" + Constants.CASCADE_DELETE + "&_pretty=true");
+		delete.addHeader(Constants.HEADER_ACCEPT, Constants.CT_FHIR_JSON_NEW);
+		ourLog.info("HttpDelete : {}", delete);
+		try (CloseableHttpResponse response = theCloseableHttpClient.execute(delete)) {
+			if (response.getStatusLine().getStatusCode() != 200) {
+				ourLog.error("Unexpected status on organization delete = " + response.getStatusLine().getStatusCode());
+				return false;
+			}
+			String deleteResponse = IOUtils.toString(response.getEntity().getContent(), Charsets.UTF_8);
+			ourLog.info("Response: {}", deleteResponse);
+			if (!deleteResponse.contains("Cascaded delete to ") && !deleteResponse.contains("Successfully deleted 1 resource(s)")) {
+				ourLog.error("Unexpected response organization delete = " + deleteResponse);
+				return false;
+			}
+		}
+
+		ourLog.info("Delete of organization resource completed.");
+
+		try {
+			ourLog.info("Reading {}", myOrganizationId);
+			myClient.read().resource(Organization.class).withId(myOrganizationId).execute();
+			fail();
+		} catch (ResourceGoneException ignored) {
+			ourLog.info("Organization resource gone as expected");
+		}
+		try {
+			ourLog.info("Searching for encounters after deleting Organization");
+			myClient.read().resource(Encounter.class).withId("Encounter/test-enc-3").execute();
+			fail();
+		} catch (ResourceGoneException ignored) {
+			ourLog.info("Encounter resource gone as expected after Organization deleted");
+		}
+		return true;
+	}
+
+}

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/CascadingDeleteInterceptorTest.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.interceptor;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.provider.r4.BaseResourceProviderR4Test;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
@@ -26,6 +27,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.transaction.PlatformTransactionManager;
 
 import java.io.IOException;
 import java.util.List;
@@ -58,6 +60,10 @@ public class CascadingDeleteInterceptorTest extends BaseResourceProviderR4Test {
 	private IIdType myEncounterId;
 	@Autowired
 	private OverridePathBasedReferentialIntegrityForDeletesInterceptor myOverridePathBasedReferentialIntegrityForDeletesInterceptor;
+	@Autowired
+	private ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
+	@Autowired
+	PlatformTransactionManager myPlatformTransactionManager;
 
 	@Override
 	@AfterEach
@@ -106,7 +112,9 @@ public class CascadingDeleteInterceptorTest extends BaseResourceProviderR4Test {
 		DaoRegistry mockDaoRegistry = mock(DaoRegistry.class);
 		IFhirResourceDao mockResourceDao = mock (IFhirResourceDao.class);
 		IBaseResource mockResource = mock(IBaseResource.class);
-		CascadingDeleteInterceptor aDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, mockDaoRegistry, myInterceptorBroadcaster);
+		// This is done in order to pass the mockDaoRegistry, otherwise this assertion will fail:  verify(mockResourceDao).read(any(IIdType.class), theRequestDetailsCaptor.capture());
+		final ThreadSafeResourceDeleterSvc threadSafeResourceDeleterSvc = new ThreadSafeResourceDeleterSvc(mockDaoRegistry, myInterceptorBroadcaster, myPlatformTransactionManager);
+		CascadingDeleteInterceptor aDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, mockDaoRegistry, myInterceptorBroadcaster, threadSafeResourceDeleterSvc);
 		ourRestServer.getInterceptorService().unregisterInterceptor(myDeleteInterceptor);
 		ourRestServer.getInterceptorService().registerInterceptor(aDeleteInterceptor);
 		when(mockDaoRegistry.getResourceDao(any(String.class))).thenReturn(mockResourceDao);

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/AuthorizationInterceptorJpaR4Test.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.provider.r4;
 
 import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.jpa.bulk.export.provider.BulkDataExportProvider;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.searchparam.matcher.AuthorizationSearchParamMatcher;
@@ -75,6 +76,8 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes
 
 	@Autowired
 	private SearchParamMatcher mySearchParamMatcher;
+	@Autowired
+	private ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
 
 	@BeforeEach
 	@Override
@@ -613,7 +616,7 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes
 
 	@Test
 	public void testDeleteCascadeBlocked() {
-		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry);
+		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry, myThreadSafeResourceDeleterSvc);
 		ourRestServer.getInterceptorService().registerInterceptor(cascadingDeleteInterceptor);
 		try {
 
@@ -657,7 +660,7 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes
 
 	@Test
 	public void testDeleteCascadeAllowed() {
-		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry);
+		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry, myThreadSafeResourceDeleterSvc);
 		ourRestServer.getInterceptorService().registerInterceptor(cascadingDeleteInterceptor);
 		try {
 
@@ -696,7 +699,7 @@ public class AuthorizationInterceptorJpaR4Test extends BaseResourceProviderR4Tes
 
 	@Test
 	public void testDeleteCascadeAllowed_ButNotOnTargetType() {
-		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry);
+		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry, myThreadSafeResourceDeleterSvc);
 		ourRestServer.getInterceptorService().registerInterceptor(cascadingDeleteInterceptor);
 		try {
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ExpungeR4Test.java

@@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.api.model.ExpungeOptions;
 import ca.uhn.fhir.jpa.api.model.ExpungeOutcome;
 import ca.uhn.fhir.jpa.dao.data.ISearchDao;
 import ca.uhn.fhir.jpa.dao.data.ISearchResultDao;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
 import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
@@ -75,6 +76,8 @@ public class ExpungeR4Test extends BaseResourceProviderR4Test {
 	private ISearchDao mySearchEntityDao;
 	@Autowired
 	private ISearchResultDao mySearchResultDao;
+	@Autowired
+	private ThreadSafeResourceDeleterSvc myThreadSafeResourceDeleterSvc;
 
 	@AfterEach
 	public void afterDisableExpunge() {
@@ -209,7 +212,7 @@ public class ExpungeR4Test extends BaseResourceProviderR4Test {
 
 	@Test
 	public void testDeleteCascade() throws IOException {
-		ourRestServer.registerInterceptor(new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry));
+		ourRestServer.registerInterceptor(new CascadingDeleteInterceptor(myFhirContext, myDaoRegistry, myInterceptorRegistry, myThreadSafeResourceDeleterSvc));
 
 		// setup
 		Organization organization = new Organization();

hapi-fhir-jpaserver-test-r5/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-utilities/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/DelayListener.java

@@ -0,0 +1,40 @@
+package ca.uhn.fhir.jpa.test.config;
+
+import net.ttddyy.dsproxy.ExecutionInfo;
+import net.ttddyy.dsproxy.QueryInfo;
+import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class DelayListener implements ProxyDataSourceBuilder.SingleQueryExecution {
+	private static final Logger ourLog = LoggerFactory.getLogger(DelayListener.class);
+
+	private boolean enabled = false;
+	private AtomicInteger deleteCount= new AtomicInteger(0);
+
+	public void enable() {
+		enabled = true;
+	}
+
+	public void reset() {
+		enabled = false;
+		deleteCount = new AtomicInteger(0);
+	}
+
+	@Override
+	public void execute(ExecutionInfo execInfo, List<QueryInfo> queryInfoList) {
+		if (enabled && queryInfoList.get(0).getQuery().contains("from HFJ_RES_LINK")) {
+			if (deleteCount.getAndIncrement() == 0) {
+				try {
+					Thread.sleep(500L);
+				} catch (InterruptedException theE) {
+					ourLog.error(theE.getMessage(), theE);
+				}
+			}
+		}
+	}
+
+}

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4WithDelayConfig.java

@@ -0,0 +1,151 @@
+package ca.uhn.fhir.jpa.test.config;
+
+/*-
+ * #%L
+ * HAPI FHIR JPA Server Test Utilities
+ * %%
+ * Copyright (C) 2014 - 2022 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.batch2.jobs.config.Batch2JobsConfig;
+import ca.uhn.fhir.jpa.batch2.JpaBatch2Config;
+import ca.uhn.fhir.jpa.config.HapiJpaConfig;
+import ca.uhn.fhir.jpa.config.r4.JpaR4Config;
+import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
+import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
+import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+@Configuration
+@Import({
+	JpaR4Config.class,
+	HapiJpaConfig.class,
+	TestJPAConfig.class,
+	TestHSearchAddInConfig.DefaultLuceneHeap.class,
+	JpaBatch2Config.class,
+	Batch2JobsConfig.class
+})
+public class TestR4WithDelayConfig extends TestR4Config {
+
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TestR4WithDelayConfig.class);
+
+	private final Deque<Exception> myLastStackTrace = new LinkedList<>();
+	private boolean myHaveDumpedThreads;
+
+	@Override
+	@Bean
+	public DataSource dataSource() {
+		BasicDataSource retVal = new BasicDataSource() {
+
+			@Override
+			public Connection getConnection() {
+				ConnectionWrapper retVal;
+				try {
+					retVal = new ConnectionWrapper(super.getConnection());
+				} catch (Exception e) {
+					ourLog.error("Exceeded maximum wait for connection (" + ourMaxThreads + " max)", e);
+					logGetConnectionStackTrace();
+					fail("Exceeded maximum wait for connection (" + ourMaxThreads + " max): " + e);
+					retVal = null;
+				}
+
+				try {
+					throw new Exception();
+				} catch (Exception e) {
+					synchronized (myLastStackTrace) {
+						myLastStackTrace.add(e);
+						while (myLastStackTrace.size() > ourMaxThreads) {
+							myLastStackTrace.removeFirst();
+						}
+					}
+				}
+
+				return retVal;
+			}
+
+			private void logGetConnectionStackTrace() {
+				StringBuilder b = new StringBuilder();
+				int i = 0;
+				synchronized (myLastStackTrace) {
+					for (Iterator<Exception> iter = myLastStackTrace.descendingIterator(); iter.hasNext(); ) {
+						Exception nextStack = iter.next();
+						b.append("\n\nPrevious request stack trace ");
+						b.append(i++);
+						b.append(":");
+						for (StackTraceElement next : nextStack.getStackTrace()) {
+							b.append("\n   ");
+							b.append(next.getClassName());
+							b.append(".");
+							b.append(next.getMethodName());
+							b.append("(");
+							b.append(next.getFileName());
+							b.append(":");
+							b.append(next.getLineNumber());
+							b.append(")");
+						}
+					}
+				}
+				ourLog.info(b.toString());
+
+				if (!myHaveDumpedThreads) {
+					ourLog.info("Thread dump:" + crunchifyGenerateThreadDump());
+					myHaveDumpedThreads = true;
+				}
+			}
+
+		};
+
+		retVal.setDriver(new org.h2.Driver());
+		retVal.setUrl("jdbc:h2:mem:testdb_r4");
+		retVal.setMaxWaitMillis(30000);
+		retVal.setUsername("");
+		retVal.setPassword("");
+		retVal.setMaxTotal(ourMaxThreads);
+
+		SLF4JLogLevel level = SLF4JLogLevel.INFO;
+		DataSource dataSource = ProxyDataSourceBuilder
+			.create(retVal)
+			.logSlowQueryBySlf4j(10, TimeUnit.SECONDS, level)
+			.beforeQuery(new BlockLargeNumbersOfParamsListener())
+			.beforeQuery(getMandatoryTransactionListener())
+			.afterQuery(captureQueriesListener())
+			.afterQuery(new CurrentThreadCaptureQueriesListener())
+			.afterQuery(delayListener())
+			.countQuery(singleQueryCountHolder())
+			.afterMethod(captureQueriesListener())
+			.build();
+
+		return dataSource;
+	}
+
+	@Bean
+	DelayListener delayListener() {
+		return new DelayListener();
+	}
+
+}

hapi-fhir-jpaserver-uhnfhirtest/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-uhnfhirtest/src/main/java/ca/uhn/fhirtest/TestRestfulServer.java

@@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
 import ca.uhn.fhir.jpa.bulk.export.provider.BulkDataExportProvider;
+import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
 import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor;
 import ca.uhn.fhir.jpa.provider.DiffProvider;
 import ca.uhn.fhir.jpa.graphql.GraphQLProvider;
@@ -262,7 +263,8 @@ public class TestRestfulServer extends RestfulServer {
 		 */
 		DaoRegistry daoRegistry = myAppCtx.getBean(DaoRegistry.class);
 		IInterceptorBroadcaster interceptorBroadcaster = myAppCtx.getBean(IInterceptorBroadcaster.class);
-		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(ctx, daoRegistry, interceptorBroadcaster);
+		ThreadSafeResourceDeleterSvc threadSafeResourceDeleterSvc = myAppCtx.getBean(ThreadSafeResourceDeleterSvc.class);
+		CascadingDeleteInterceptor cascadingDeleteInterceptor = new CascadingDeleteInterceptor(ctx, daoRegistry, interceptorBroadcaster, threadSafeResourceDeleterSvc);
 		registerInterceptor(cascadingDeleteInterceptor);
 
 		/*

hapi-fhir-server-mdm/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server-openapi/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/TransactionDetails.java

@@ -61,7 +61,6 @@ public class TransactionDetails {
 	private Map<String, Object> myUserData;
 	private ListMultimap<Pointcut, HookParams> myDeferredInterceptorBroadcasts;
 	private EnumSet<Pointcut> myDeferredInterceptorBroadcastPointcuts;
-	private boolean myIsPointcutDeferred;
 
 	/**
 	 * Constructor
@@ -273,7 +272,6 @@ public class TransactionDetails {
 	 */
 	public void addDeferredInterceptorBroadcast(Pointcut thePointcut, HookParams theHookParams) {
 		Validate.isTrue(isAcceptingDeferredInterceptorBroadcasts(thePointcut));
-		myIsPointcutDeferred = true;
 		myDeferredInterceptorBroadcasts.put(thePointcut, theHookParams);
 	}
 
@@ -286,7 +284,6 @@ public class TransactionDetails {
 	}
 
 	public void deferredBroadcastProcessingFinished() {
-		myIsPointcutDeferred = false;
 	}
 
 	public void clearResolvedItems() {

hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-apache</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-okhttp</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-server-jersey</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-samples</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml

@@ -5,7 +5,7 @@
    <parent>
       <groupId>ca.uhn.hapi.fhir</groupId>
       <artifactId>hapi-deployable-pom</artifactId>
-      <version>6.2.0-PRE12-SNAPSHOT</version>
+      <version>6.2.0-PRE13-SNAPSHOT</version>
       <relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
    </parent>
 

hapi-fhir-spring-boot/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-sql-migrate/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage-batch2-jobs/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage-batch2/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage-mdm/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage-test-utilities/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java

@@ -41,7 +41,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.support.TransactionCallback;
-import org.springframework.transaction.support.TransactionCallbackWithoutResult;
 import org.springframework.transaction.support.TransactionTemplate;
 
 import javax.annotation.Nullable;
@@ -158,6 +157,10 @@ public class HapiTransactionService {
 		}
 	}
 
+	public PlatformTransactionManager getTransactionManager() {
+		return myTransactionManager;
+	}
+
 	/**
 	 * This is just an unchecked exception so that we can catch checked exceptions inside TransactionTemplate
 	 * and rethrow them outside of it

hapi-fhir-structures-dstu2.1/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu2/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu3/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-hl7org-dstu2/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-r4/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-r5/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-test-utilities/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-testpage-overlay/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu2.1/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu2/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu3/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-r4/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-r5/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-tinder-plugin/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 
@@ -65,37 +65,37 @@
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-structures-dstu3</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-structures-hl7org-dstu2</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-structures-r4</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-structures-r5</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-validation-resources-dstu2</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-validation-resources-dstu3</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-validation-resources-r4</artifactId>
-			<version>6.2.0-PRE12-SNAPSHOT</version>
+			<version>6.2.0-PRE13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.velocity</groupId>

hapi-tinder-test/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

pom.xml

@@ -6,7 +6,7 @@
 	<groupId>ca.uhn.hapi.fhir</groupId>
 	<artifactId>hapi-fhir</artifactId>
 	<packaging>pom</packaging>
-	<version>6.2.0-PRE12-SNAPSHOT</version>
+	<version>6.2.0-PRE13-SNAPSHOT</version>
 	<name>HAPI-FHIR</name>
 	<description>An open-source implementation of the FHIR specification in Java.</description>
 	<url>https://hapifhir.io</url>
@@ -2014,7 +2014,7 @@
                             <groupId>ca.uhn.hapi.fhir</groupId>
                             <artifactId>hapi-fhir-checkstyle</artifactId>
 									<!-- Remember to bump this when you upgrade the version -->
-                            <version>6.2.0-PRE12-SNAPSHOT</version>
+                            <version>6.2.0-PRE13-SNAPSHOT</version>
                         </dependency>
 					</dependencies>
 				</plugin>

tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 

tests/hapi-fhir-base-test-mindeps-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 

tests/hapi-fhir-base-test-mindeps-server/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.2.0-PRE12-SNAPSHOT</version>
+		<version>6.2.0-PRE13-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>