Streamline expunge operation

hapi-fhir-cli/hapi-fhir-cli-api/pom.xml

@@ -188,7 +188,7 @@
 		</dependency>
 		<dependency>
 			<groupId>org.thymeleaf</groupId>
-			<artifactId>thymeleaf-spring4</artifactId>
+			<artifactId>thymeleaf-spring5</artifactId>
 		</dependency>
 
 		<!-- Dependencies for Schematron -->

hapi-fhir-jpaserver-base/pom.xml

@@ -163,7 +163,7 @@
 		</dependency>
 		<dependency>
 			<groupId>org.thymeleaf</groupId>
-			<artifactId>thymeleaf-spring4</artifactId>
+			<artifactId>thymeleaf-spring5</artifactId>
 		</dependency>
 
 		<!-- For UCUM: TODO we should replace this with org.fhir UCUM -->

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java

@@ -42,6 +42,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
 import org.springframework.core.env.Environment;
+import org.springframework.core.task.AsyncTaskExecutor;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 import org.springframework.orm.hibernate5.HibernateExceptionTranslator;
 import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
@@ -162,7 +163,7 @@ public abstract class BaseConfig implements SchedulingConfigurer {
 		return new SubscriptionWebsocketInterceptor();
 	}
 
-	@Bean(name = TASK_EXECUTOR_NAME)
+	@Bean()
 	public TaskScheduler taskScheduler() {
 		ConcurrentTaskScheduler retVal = new ConcurrentTaskScheduler();
 		retVal.setConcurrentExecutor(scheduledExecutorService());
@@ -170,6 +171,14 @@ public abstract class BaseConfig implements SchedulingConfigurer {
 		return retVal;
 	}
 
+	@Bean(name = TASK_EXECUTOR_NAME)
+	public AsyncTaskExecutor taskExecutor() {
+		ConcurrentTaskScheduler retVal = new ConcurrentTaskScheduler();
+		retVal.setConcurrentExecutor(scheduledExecutorService());
+		retVal.setScheduledExecutor(scheduledExecutorService());
+		return retVal;
+	}
+
 	@Bean
 	public IResourceReindexingSvc resourceReindexingSvc() {
 		return new ResourceReindexingSvcImpl();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -37,6 +37,7 @@ import javax.xml.stream.events.Characters;
 import javax.xml.stream.events.XMLEvent;
 
 import ca.uhn.fhir.jpa.dao.data.*;
+import com.google.common.collect.Lists;
 import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.lang3.Validate;
 import org.apache.http.NameValuePair;
@@ -255,6 +256,8 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 	protected IResourceIndexedSearchParamStringDao myResourceIndexedSearchParamStringDao;
 	@Autowired()
 	protected IResourceIndexedSearchParamTokenDao myResourceIndexedSearchParamTokenDao;
+	@Autowired
+	protected IResourceLinkDao myResourceLinkDao;
 	@Autowired()
 	protected IResourceIndexedSearchParamDateDao myResourceIndexedSearchParamDateDao;
 	@Autowired()
@@ -328,6 +331,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 
 	protected ExpungeOutcome doExpunge(String theResourceName, Long theResourceId, Long theVersion, ExpungeOptions theExpungeOptions) {
 		TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager);
+		txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
 		ourLog.info("Expunge: ResourceName[{}] Id[{}] Version[{}] Options[{}]", theResourceName, theResourceId, theVersion, theExpungeOptions);
 
 		if (!getConfig().isExpungeEnabled()) {
@@ -367,11 +371,14 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 			});
 
 			/*
-			 * Delete any search result cache entries pointing to the given resource
+			 * Delete any search result cache entries pointing to the given resource. We do
+			 * this in batches to avoid sending giant batches of parameters to the DB
 			 */
-			if (resourceIds.getContent().size() > 0) {
+			List<List<Long>> partitions = Lists.partition(resourceIds.getContent(), 800);
+			for (List<Long> nextPartition : partitions) {
+				ourLog.info("Expunging any search results pointing to {} resources", nextPartition.size());
 				txTemplate.execute(t -> {
-					mySearchResultDao.deleteByResourceIds(resourceIds.getContent());
+					mySearchResultDao.deleteByResourceIds(nextPartition);
 					return null;
 				});
 			}
@@ -438,7 +445,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 
 		ourLog.info("** BEGINNING GLOBAL $expunge **");
 		TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager);
-		txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRED);
+		txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
 		txTemplate.execute(t -> {
 			doExpungeEverythingQuery("UPDATE " + ResourceHistoryTable.class.getSimpleName() + " d SET d.myForcedId = null");
 			doExpungeEverythingQuery("UPDATE " + ResourceTable.class.getSimpleName() + " d SET d.myForcedId = null");
@@ -521,6 +528,8 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 		myResourceIndexedSearchParamQuantityDao.deleteAll(resource.getParamsQuantity());
 		myResourceIndexedSearchParamStringDao.deleteAll(resource.getParamsString());
 		myResourceIndexedSearchParamTokenDao.deleteAll(resource.getParamsToken());
+		myResourceLinkDao.deleteAll(resource.getResourceLinks());
+		myResourceLinkDao.deleteAll(resource.getResourceLinksAsTarget());
 
 		myResourceTagDao.deleteAll(resource.getTags());
 		resource.getTags().clear();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java

@@ -65,7 +65,7 @@ public abstract class BaseHapiFhirSystemDao<T, MT> extends BaseHapiFhirDao<IBase
 
 
 	@Override
-	@Transactional(propagation = Propagation.REQUIRED)
+	@Transactional(propagation = Propagation.NEVER)
 	public ExpungeOutcome expunge(ExpungeOptions theExpungeOptions) {
 		return doExpunge(null, null, null, theExpungeOptions);
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceLinkDao.java

@@ -25,5 +25,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
 import ca.uhn.fhir.jpa.entity.ResourceLink;
 
 public interface IResourceLinkDao  extends JpaRepository<ResourceLink, Long> {
-	// nothing
+
+
+
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceTable.java

@@ -185,27 +185,33 @@ public class ResourceTable extends BaseHasResource implements Serializable {
 	@IndexedEmbedded()
 	@OptimisticLock(excluded = true)
 	private Collection<ResourceLink> myResourceLinks;
-
+	@OneToMany(mappedBy = "myTargetResource", cascade = {}, fetch = FetchType.LAZY, orphanRemoval = false)
+	@IndexedEmbedded()
+	@OptimisticLock(excluded = true)
+	private Collection<ResourceLink> myResourceLinksAsTarget;
 	@Column(name = "RES_TYPE", length = RESTYPE_LEN)
 	@Field
 	@OptimisticLock(excluded = true)
 	private String myResourceType;
-
 	@OneToMany(mappedBy = "myResource", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true)
 	@OptimisticLock(excluded = true)
 	private Collection<SearchParamPresent> mySearchParamPresents;
-
 	@OneToMany(mappedBy = "myResource", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true)
 	@OptimisticLock(excluded = true)
 	private Set<ResourceTag> myTags;
-
 	@Transient
 	private transient boolean myUnchangedInCurrentOperation;
-
 	@Version
 	@Column(name = "RES_VER")
 	private long myVersion;
 
+	public Collection<ResourceLink> getResourceLinksAsTarget() {
+		if (myResourceLinksAsTarget == null) {
+			myResourceLinksAsTarget = new ArrayList<>();
+		}
+		return myResourceLinksAsTarget;
+	}
+
 	@Override
 	public ResourceTag addTag(TagDefinition theTag) {
 		for (ResourceTag next : getTags()) {

hapi-fhir-testpage-overlay/pom.xml

@@ -67,7 +67,7 @@
 		</dependency>
 		<dependency>
 			<groupId>org.thymeleaf</groupId>
-			<artifactId>thymeleaf-spring4</artifactId>
+			<artifactId>thymeleaf-spring5</artifactId>
 		</dependency>
 		<dependency>
 			<groupId>javax.servlet</groupId>

pom.xml

@@ -518,9 +518,9 @@
 		<jsr305_version>3.0.2</jsr305_version>
 		<!--<hibernate_version>5.2.10.Final</hibernate_version>-->
 		<hibernate_version>5.3.6.Final</hibernate_version>
+		<hibernate_search_version>5.10.3.Final</hibernate_search_version>
 		<hibernate_validator_version>5.4.1.Final</hibernate_validator_version>
 		<!-- Update lucene version when you update hibernate-search version -->
-		<hibernate_search_version>5.10.3.Final</hibernate_search_version>
 		<httpcore_version>4.4.6</httpcore_version>
 		<httpclient_version>4.5.3</httpclient_version>
 		<jackson_version>2.9.7</jackson_version>
@@ -1246,7 +1246,7 @@
 			</dependency>
 			<dependency>
 				<groupId>org.thymeleaf</groupId>
-				<artifactId>thymeleaf-spring4</artifactId>
+				<artifactId>thymeleaf-spring5</artifactId>
 				<version>${thymeleaf-version}</version>
 			</dependency>
 			<dependency>

src/changes/changes.xml

@@ -6,6 +6,25 @@
 		<title>HAPI FHIR Changelog</title>
 	</properties>
 	<body>
+		<release version="3.7.0" date="TBD" description="Gale">
+			<action type="add">
+				The version of a few dependencies have been bumped to the
+				latest versions (dependent HAPI modules listed in brackets):
+				<![CDATA[
+					<ul>
+						<li>thymeleaf-spring4 (Testpage Overlay) has been replaced with thymeleaf-spring5</li>
+					</ul>
+				]]>
+			</action>
+			<action type="fix">
+				The JPA server $expunge operation could sometimes fail to expunge if
+				another resource linked to a resource that was being
+				expunged. This has been corrected. In addition, the $expunge operation 
+				has been refactored to use smaller chunks of work
+				within a single DB transaction. This improves performance and reduces contention when
+				performing large expunge workloads.
+			</action>
+		</release>
 		<release version="3.6.0" date="2018-11-12" description="Food">
 			<action type="add">
 				The version of a few dependencies have been bumped to the