From 84acafe3af7170d1a7d272f56aad6850effa56f6 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 15 Nov 2018 11:35:50 +0100 Subject: [PATCH] Streamline expunge operation --- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 2 +- hapi-fhir-jpaserver-base/pom.xml | 2 +- .../ca/uhn/fhir/jpa/config/BaseConfig.java | 11 +++++++++- .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 17 ++++++++++++---- .../fhir/jpa/dao/BaseHapiFhirSystemDao.java | 2 +- .../fhir/jpa/dao/data/IResourceLinkDao.java | 4 +++- .../ca/uhn/fhir/jpa/entity/ResourceTable.java | 20 ++++++++++++------- hapi-fhir-testpage-overlay/pom.xml | 2 +- pom.xml | 4 ++-- src/changes/changes.xml | 19 ++++++++++++++++++ 10 files changed, 64 insertions(+), 19 deletions(-) diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index fc7c270042f..1ffeda550f2 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -188,7 +188,7 @@ org.thymeleaf - thymeleaf-spring4 + thymeleaf-spring5 diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 0ae95813b59..125398d7113 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -163,7 +163,7 @@ org.thymeleaf - thymeleaf-spring4 + thymeleaf-spring5 diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 9667e2ba924..235eeca6adf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -42,6 +42,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.core.env.Environment; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.orm.hibernate5.HibernateExceptionTranslator; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; @@ -162,7 +163,7 @@ public abstract class BaseConfig implements SchedulingConfigurer { return new SubscriptionWebsocketInterceptor(); } - @Bean(name = TASK_EXECUTOR_NAME) + @Bean() public TaskScheduler taskScheduler() { ConcurrentTaskScheduler retVal = new ConcurrentTaskScheduler(); retVal.setConcurrentExecutor(scheduledExecutorService()); @@ -170,6 +171,14 @@ public abstract class BaseConfig implements SchedulingConfigurer { return retVal; } + @Bean(name = TASK_EXECUTOR_NAME) + public AsyncTaskExecutor taskExecutor() { + ConcurrentTaskScheduler retVal = new ConcurrentTaskScheduler(); + retVal.setConcurrentExecutor(scheduledExecutorService()); + retVal.setScheduledExecutor(scheduledExecutorService()); + return retVal; + } + @Bean public IResourceReindexingSvc resourceReindexingSvc() { return new ResourceReindexingSvcImpl(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index da6135a5456..846eefe5d84 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -37,6 +37,7 @@ import javax.xml.stream.events.Characters; import javax.xml.stream.events.XMLEvent; import ca.uhn.fhir.jpa.dao.data.*; +import com.google.common.collect.Lists; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.Validate; import org.apache.http.NameValuePair; @@ -255,6 +256,8 @@ public abstract class BaseHapiFhirDao implements IDao, protected IResourceIndexedSearchParamStringDao myResourceIndexedSearchParamStringDao; @Autowired() protected IResourceIndexedSearchParamTokenDao myResourceIndexedSearchParamTokenDao; + @Autowired + protected IResourceLinkDao myResourceLinkDao; @Autowired() protected IResourceIndexedSearchParamDateDao myResourceIndexedSearchParamDateDao; @Autowired() @@ -328,6 +331,7 @@ public abstract class BaseHapiFhirDao implements IDao, protected ExpungeOutcome doExpunge(String theResourceName, Long theResourceId, Long theVersion, ExpungeOptions theExpungeOptions) { TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager); + txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW); ourLog.info("Expunge: ResourceName[{}] Id[{}] Version[{}] Options[{}]", theResourceName, theResourceId, theVersion, theExpungeOptions); if (!getConfig().isExpungeEnabled()) { @@ -367,11 +371,14 @@ public abstract class BaseHapiFhirDao implements IDao, }); /* - * Delete any search result cache entries pointing to the given resource + * Delete any search result cache entries pointing to the given resource. We do + * this in batches to avoid sending giant batches of parameters to the DB */ - if (resourceIds.getContent().size() > 0) { + List> partitions = Lists.partition(resourceIds.getContent(), 800); + for (List nextPartition : partitions) { + ourLog.info("Expunging any search results pointing to {} resources", nextPartition.size()); txTemplate.execute(t -> { - mySearchResultDao.deleteByResourceIds(resourceIds.getContent()); + mySearchResultDao.deleteByResourceIds(nextPartition); return null; }); } @@ -438,7 +445,7 @@ public abstract class BaseHapiFhirDao implements IDao, ourLog.info("** BEGINNING GLOBAL $expunge **"); TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager); - txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRED); + txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW); txTemplate.execute(t -> { doExpungeEverythingQuery("UPDATE " + ResourceHistoryTable.class.getSimpleName() + " d SET d.myForcedId = null"); doExpungeEverythingQuery("UPDATE " + ResourceTable.class.getSimpleName() + " d SET d.myForcedId = null"); @@ -521,6 +528,8 @@ public abstract class BaseHapiFhirDao implements IDao, myResourceIndexedSearchParamQuantityDao.deleteAll(resource.getParamsQuantity()); myResourceIndexedSearchParamStringDao.deleteAll(resource.getParamsString()); myResourceIndexedSearchParamTokenDao.deleteAll(resource.getParamsToken()); + myResourceLinkDao.deleteAll(resource.getResourceLinks()); + myResourceLinkDao.deleteAll(resource.getResourceLinksAsTarget()); myResourceTagDao.deleteAll(resource.getTags()); resource.getTags().clear(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java index 830d5abbfc1..c37c458b0c2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java @@ -65,7 +65,7 @@ public abstract class BaseHapiFhirSystemDao extends BaseHapiFhirDao { - // nothing + + + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceTable.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceTable.java index e30a9b83438..891c31641b8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceTable.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceTable.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.entity; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -185,27 +185,33 @@ public class ResourceTable extends BaseHasResource implements Serializable { @IndexedEmbedded() @OptimisticLock(excluded = true) private Collection myResourceLinks; - + @OneToMany(mappedBy = "myTargetResource", cascade = {}, fetch = FetchType.LAZY, orphanRemoval = false) + @IndexedEmbedded() + @OptimisticLock(excluded = true) + private Collection myResourceLinksAsTarget; @Column(name = "RES_TYPE", length = RESTYPE_LEN) @Field @OptimisticLock(excluded = true) private String myResourceType; - @OneToMany(mappedBy = "myResource", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) @OptimisticLock(excluded = true) private Collection mySearchParamPresents; - @OneToMany(mappedBy = "myResource", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) @OptimisticLock(excluded = true) private Set myTags; - @Transient private transient boolean myUnchangedInCurrentOperation; - @Version @Column(name = "RES_VER") private long myVersion; + public Collection getResourceLinksAsTarget() { + if (myResourceLinksAsTarget == null) { + myResourceLinksAsTarget = new ArrayList<>(); + } + return myResourceLinksAsTarget; + } + @Override public ResourceTag addTag(TagDefinition theTag) { for (ResourceTag next : getTags()) { diff --git a/hapi-fhir-testpage-overlay/pom.xml b/hapi-fhir-testpage-overlay/pom.xml index bdc30afcf4d..d377521707b 100644 --- a/hapi-fhir-testpage-overlay/pom.xml +++ b/hapi-fhir-testpage-overlay/pom.xml @@ -67,7 +67,7 @@ org.thymeleaf - thymeleaf-spring4 + thymeleaf-spring5 javax.servlet diff --git a/pom.xml b/pom.xml index cdbeb43ae80..38c44e721ab 100644 --- a/pom.xml +++ b/pom.xml @@ -518,9 +518,9 @@ 3.0.2 5.3.6.Final + 5.10.3.Final 5.4.1.Final - 5.10.3.Final 4.4.6 4.5.3 2.9.7 @@ -1246,7 +1246,7 @@ org.thymeleaf - thymeleaf-spring4 + thymeleaf-spring5 ${thymeleaf-version} diff --git a/src/changes/changes.xml b/src/changes/changes.xml index e4199967673..f73f46251e1 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -6,6 +6,25 @@ HAPI FHIR Changelog + + + The version of a few dependencies have been bumped to the + latest versions (dependent HAPI modules listed in brackets): + +
  • thymeleaf-spring4 (Testpage Overlay) has been replaced with thymeleaf-spring5
    • + + ]]> +     + + The JPA server $expunge operation could sometimes fail to expunge if + another resource linked to a resource that was being + expunged. This has been corrected. In addition, the $expunge operation + has been refactored to use smaller chunks of work + within a single DB transaction. This improves performance and reduces contention when + performing large expunge workloads. + +     The version of a few dependencies have been bumped to the