honeymoose/hapi-fhir
1
0
Fork 0
mirror of https://github.com/hapifhir/hapi-fhir.git synced 2025-03-27 10:28:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Issue 3108 terminology large codesystem deletion takes days to complete (#3160)

Browse Source 
* Add indexes to foreign keys

* Add new version enum

* Add TermCodeSystem and TermCodeSystemVersion deletion job artifacts

* Add Propagation.NOT_SUPPORTED to allow starting jobs from transactional code

* New jobs artifacts

* Improve comments

* Restructure job configurations to avoid bean name collisions

* Restructure job configurations to avoid bean name collisions

* Use new offline deletion by job

* Test using multiple versions

* Retrieve only Pids

* Revert to master

* Maintain deferred functions synchronization as similar as possible to the way it was

* Bypass validations to allow calling an index same as a foreign key to make SchemaMigrationTest happy as H2 adds FK indexes automatically

* Adjust tests

* Add copyrights

* Add test

* Revert to delete CodeSystemVersions using deferred ITermDeferredStorageSvc

* Allow to wait for jobs tentatively

* Organize imports

* Fix tests

* Increase timeout to cover for possible additional heavy DB activity

* Add indexes to two remaining non-indexed foreign key fields to avoid much longer transactions which produce locks on heavy DB load

* Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3108-terminology-large-code-system-deletion-performance.yaml

Co-authored-by: James Agnew <jamesagnew@gmail.com>

* Remove opening stars from logging messages

* Relax index name checking logic to allow for indexes on FK columns to be called same as the FKs, to make SchemaMigrationTest happy even when H2 adds indexes to FKs automatically.

* Use static array to enumerate platforms which doesn't add indexes to foreign keys automatically.

* Cleanup also job executions when performing test artifact cleanup

* Adjust to property name change

* Add ABANDONED status to the list of waited job statuses to avoid test failures due to wait timeouts

* Add also UNKNOWN status to the list of waited job statuses to avoid test failures due to wait timeouts

* Filter job executions to be stopped

* Set trace to debug test problem

* Add STOPPED status to the list of waited job statuses to avoid test failures due to wait timeouts

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>
Co-authored-by: James Agnew <jamesagnew@gmail.com>
This commit is contained in:
jmarchionatto 2021-11-18 09:49:16 -05:00 committed by GitHub
parent 0c3fb775df
commit 175958de5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 1891 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/config/BatchConstants.java
View File

@ -54,6 +54,31 @@ public final class BatchConstants {
* MDM Clear
*/
public static final String MDM_CLEAR_JOB_NAME = "mdmClearJob";
/**
* TermCodeSystem delete
*/
public static final String TERM_CODE_SYSTEM_DELETE_JOB_NAME = "termCodeSystemDeleteJob";
public static final String TERM_CONCEPT_RELATIONS_DELETE_STEP_NAME = "termConceptRelationsDeleteStep";
public static final String TERM_CONCEPTS_DELETE_STEP_NAME = "termConceptsDeleteStep";
public static final String TERM_CODE_SYSTEM_VERSION_DELETE_STEP_NAME = "termCodeSystemVersionDeleteStep";
public static final String TERM_CODE_SYSTEM_DELETE_STEP_NAME = "termCodeSystemDeleteStep";
public static final String JOB_PARAM_CODE_SYSTEM_ID = "termCodeSystemPid";
/**
* TermCodeSystemVersion delete
*/
public static final String TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME = "termCodeSystemVersionDeleteJob";
public static final String TERM_CONCEPT_RELATIONS_UNIQUE_VERSION_DELETE_STEP_NAME = "termConceptRelationsUniqueVersionDeleteStep";
public static final String TERM_CONCEPTS_UNIQUE_VERSION_DELETE_STEP_NAME = "termConceptsUniqueVersionDeleteStep";
public static final String TERM_CODE_SYSTEM_UNIQUE_VERSION_DELETE_STEP_NAME = "termCodeSystemUniqueVersionDeleteStep";
/**
* Both: TermCodeSystem delete and TermCodeSystemVersion delete
*/
public static final String JOB_PARAM_CODE_SYSTEM_VERSION_ID = "termCodeSystemVersionPid";
public static final String BULK_EXPORT_READ_CHUNK_PARAMETER = "readChunkSize";
public static final String BULK_EXPORT_GROUP_ID_PARAMETER = "groupId";
/**

3
hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/svc/BatchJobSubmitterImpl.java
View File

@ -32,6 +32,8 @@ import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteExcep
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import static org.slf4j.LoggerFactory.getLogger;
@ -46,6 +48,7 @@ public class BatchJobSubmitterImpl implements IBatchJobSubmitter {
private JobRepository myJobRepository;
@Override
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public JobExecution runJob(Job theJob, JobParameters theJobParameters) throws JobParametersInvalidException {
try {
return myJobLauncher.run(theJob, theJobParameters);

5
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3108-terminology-large-code-system-deletion-performance.yaml Normal file
View File

@ -0,0 +1,5 @@
---
type: fix
issue: 3108
title: "Code System deletion background tasks were taking over a day to complete on very large CodeSystems for PostgreSQL, SQL Server and Oracle
databases. That was improved now taking less than an hour in all three platforms"

19
hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java
View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.reflect.ClassPath;
import com.google.common.reflect.ClassPath.ClassInfo;
import org.apache.commons.io.IOUtils;
@ -74,6 +75,17 @@ public class TestUtil {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TestUtil.class);
private static Set<String> ourReservedWords;
// Exceptions set because H2 sets indexes for FKs automatically so this index had to be called as the target FK field
// it is indexing to avoid SchemaMigrationTest to complain about the extra index (which doesn't exist in H2)
private static final Set<String> duplicateNameValidationExceptionList = Sets.newHashSet(
"FK_CONCEPTPROP_CONCEPT",
"FK_CONCEPTDESIG_CONCEPT",
"FK_TERM_CONCEPTPC_CHILD",
"FK_TERM_CONCEPTPC_PARENT"
);
/**
* non instantiable
*/
@ -252,7 +264,8 @@ public class TestUtil {
}
for (Index nextConstraint : table.indexes()) {
assertNotADuplicateName(nextConstraint.name(), theNames);
Validate.isTrue(nextConstraint.name().startsWith("IDX_"), nextConstraint.name() + " must start with IDX_");
Validate.isTrue(nextConstraint.name().startsWith("IDX_") || nextConstraint.name().startsWith("FK_"),
nextConstraint.name() + " must start with IDX_ or FK_ (last one when indexing a FK column)");
}
}
@ -269,7 +282,9 @@ public class TestUtil {
Validate.notNull(fk);
Validate.isTrue(isNotBlank(fk.name()), "Foreign key on " + theAnnotatedElement + " has no name()");
Validate.isTrue(fk.name().startsWith("FK_"));
assertNotADuplicateName(fk.name(), theNames);
if ( ! duplicateNameValidationExceptionList.contains(fk.name())) {
assertNotADuplicateName(fk.name(), theNames);
}
}
}

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java
View File

@ -26,6 +26,8 @@ import ca.uhn.fhir.jpa.bulk.imprt.job.BulkImportJobConfig;
import ca.uhn.fhir.jpa.delete.job.DeleteExpungeJobConfig;
import ca.uhn.fhir.jpa.reindex.job.ReindexEverythingJobConfig;
import ca.uhn.fhir.jpa.reindex.job.ReindexJobConfig;
import ca.uhn.fhir.jpa.term.job.TermCodeSystemDeleteJobConfig;
import ca.uhn.fhir.jpa.term.job.TermCodeSystemVersionDeleteJobConfig;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
@ -38,7 +40,9 @@ import org.springframework.context.annotation.Import;
DeleteExpungeJobConfig.class,
ReindexJobConfig.class,
ReindexEverythingJobConfig.class,
MdmClearJobConfig.class
MdmClearJobConfig.class,
TermCodeSystemDeleteJobConfig.class,
TermCodeSystemVersionDeleteJobConfig.class
})
public class BatchJobsConfig {
}

20
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java
View File

@ -155,8 +155,13 @@ import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValid
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.utilities.graphql.IGraphQLStorageServices;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.configuration.annotation.BatchConfigurer;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.support.SimpleJobOperator;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
import org.springframework.context.ApplicationContext;
@ -229,6 +234,9 @@ public abstract class BaseConfig {
private Integer searchCoordMaxPoolSize = 100;
private Integer searchCoordQueueCapacity = 200;
@Autowired
private JobLauncher myJobLauncher;
/**
* Subclasses may override this method to provide settings such as search coordinator pool sizes.
*/
@ -279,6 +287,18 @@ public abstract class BaseConfig {
return new CascadingDeleteInterceptor(theFhirContext, theDaoRegistry, theInterceptorBroadcaster);
}
@Bean
public SimpleJobOperator jobOperator(JobExplorer jobExplorer, JobRepository jobRepository, JobRegistry jobRegistry) {
SimpleJobOperator jobOperator = new SimpleJobOperator();
jobOperator.setJobExplorer(jobExplorer);
jobOperator.setJobRepository(jobRepository);
jobOperator.setJobRegistry(jobRegistry);
jobOperator.setJobLauncher(myJobLauncher);
return jobOperator;
}
@Lazy
@Bean

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermCodeSystemVersionDao.java
View File

@ -35,8 +35,8 @@ public interface ITermCodeSystemVersionDao extends JpaRepository<TermCodeSystemV
@Query("DELETE FROM TermCodeSystemVersion csv WHERE csv.myCodeSystem = :cs")
void deleteForCodeSystem(@Param("cs") TermCodeSystem theCodeSystem);
@Query("SELECT cs FROM TermCodeSystemVersion cs WHERE cs.myCodeSystemPid = :codesystem_pid")
List<TermCodeSystemVersion> findByCodeSystemPid(@Param("codesystem_pid") Long theCodeSystemPid);
@Query("SELECT myId FROM TermCodeSystemVersion WHERE myCodeSystemPid = :codesystem_pid order by myId")
List<Long> findSortedPidsByCodeSystemPid(@Param("codesystem_pid") Long theCodeSystemPid);
@Query("SELECT cs FROM TermCodeSystemVersion cs WHERE cs.myCodeSystemPid = :codesystem_pid AND cs.myCodeSystemVersionId = :codesystem_version_id")
TermCodeSystemVersion findByCodeSystemPidAndVersion(@Param("codesystem_pid") Long theCodeSystemPid, @Param("codesystem_version_id") String theCodeSystemVersionId);

7
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java
View File

@ -4,8 +4,8 @@ import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@ -40,8 +40,9 @@ public interface ITermConceptDao extends JpaRepository<TermConcept, Long>, IHapi
@Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system AND c.myCode = :code")
Optional<TermConcept> findByCodeSystemAndCode(@Param("code_system") TermCodeSystemVersion theCodeSystem, @Param("code") String theCode);
@Query("SELECT t.myId FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid")
Slice<Long> findIdsByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid);
@Modifying
@Query("DELETE FROM TermConcept WHERE myCodeSystem.myId = :cs_pid")
int deleteByCodeSystemVersion(@Param("cs_pid") Long thePid);
@Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system")
List<TermConcept> findByCodeSystemVersion(@Param("code_system") TermCodeSystemVersion theCodeSystem);

11
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDesignationDao.java
View File

@ -1,9 +1,8 @@
package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@ -29,10 +28,8 @@ import org.springframework.data.repository.query.Param;
public interface ITermConceptDesignationDao extends JpaRepository<TermConceptDesignation, Long>, IHapiFhirJpaRepository {
@Query("SELECT t.myId FROM TermConceptDesignation t WHERE t.myCodeSystemVersion.myId = :csv_pid")
Slice<Long> findIdsByCodeSystemVersion(Pageable thePage, @Param("csv_pid") Long thePid);
@Query("SELECT COUNT(t) FROM TermConceptDesignation t WHERE t.myCodeSystemVersion.myId = :csv_pid")
Integer countByCodeSystemVersion(@Param("csv_pid") Long thePid);
@Modifying
@Query("DELETE FROM TermConceptDesignation WHERE myCodeSystemVersion.myId = :csv_pid")
int deleteByCodeSystemVersion(@Param("csv_pid") Long thePid);
}

11
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java
View File

@ -1,9 +1,8 @@
package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@ -31,13 +30,11 @@ import java.util.Collection;
public interface ITermConceptParentChildLinkDao extends JpaRepository<TermConceptParentChildLink, Long>, IHapiFhirJpaRepository {
@Query("SELECT COUNT(t) FROM TermConceptParentChildLink t WHERE t.myCodeSystem.myId = :cs_pid")
Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid);
@Query("SELECT t.myParentPid FROM TermConceptParentChildLink t WHERE t.myChildPid = :child_pid")
Collection<Long> findAllWithChild(@Param("child_pid") Long theConceptPid);
@Query("SELECT t.myPid FROM TermConceptParentChildLink t WHERE t.myCodeSystem.myId = :cs_pid")
Slice<Long> findIdsByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid);
@Modifying
@Query("DELETE FROM TermConceptParentChildLink WHERE myCodeSystemVersionPid = :cs_pid")
int deleteByCodeSystemVersion(@Param("cs_pid") Long thePid);
}

8
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptPropertyDao.java
View File

@ -1,9 +1,8 @@
package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.TermConceptProperty;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@ -29,8 +28,9 @@ import org.springframework.data.repository.query.Param;
public interface ITermConceptPropertyDao extends JpaRepository<TermConceptProperty, Long>, IHapiFhirJpaRepository {
@Query("SELECT t.myId FROM TermConceptProperty t WHERE t.myCodeSystemVersion.myId = :cs_pid")
Slice<Long> findIdsByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid);
@Modifying
@Query("DELETE FROM TermConceptProperty WHERE myCodeSystemVersion.myId = :cs_pid")
int deleteByCodeSystemVersion(@Param("cs_pid") Long thePid);
@Query("SELECT COUNT(t) FROM TermConceptProperty t WHERE t.myCodeSystemVersion.myId = :cs_pid")
Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid);

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptDesignation.java
View File

@ -30,8 +30,9 @@ import static org.apache.commons.lang3.StringUtils.left;
import static org.apache.commons.lang3.StringUtils.length;
@Entity
@Table(name = "TRM_CONCEPT_DESIG", uniqueConstraints = {
}, indexes = {
@Table(name = "TRM_CONCEPT_DESIG", uniqueConstraints = { }, indexes = {
// must have same name that indexed FK or SchemaMigrationTest complains because H2 sets this index automatically
@Index(name = "FK_CONCEPTDESIG_CONCEPT", columnList = "CONCEPT_PID", unique = false)
})
public class TermConceptDesignation implements Serializable {
private static final long serialVersionUID = 1L;

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java
View File

@ -27,6 +27,9 @@ import java.io.Serializable;
@Entity
@Table(name = "TRM_CONCEPT_PC_LINK", indexes = {
// must have same name that indexed FK or SchemaMigrationTest complains because H2 sets this index automatically
@Index(name = "FK_TERM_CONCEPTPC_CHILD", columnList = "CHILD_PID", unique = false),
@Index(name = "FK_TERM_CONCEPTPC_PARENT", columnList = "PARENT_PID", unique = false)
})
public class TermConceptParentChildLink implements Serializable {
private static final long serialVersionUID = 1L;

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptProperty.java
View File

@ -36,6 +36,7 @@ import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.Lob;
import javax.persistence.ManyToOne;
@ -48,7 +49,9 @@ import static org.apache.commons.lang3.StringUtils.left;
import static org.apache.commons.lang3.StringUtils.length;
@Entity
@Table(name = "TRM_CONCEPT_PROPERTY", uniqueConstraints = {
@Table(name = "TRM_CONCEPT_PROPERTY", uniqueConstraints = { }, indexes = {
// must have same name that indexed FK or SchemaMigrationTest complains because H2 sets this index automatically
@Index(name = "FK_CONCEPTPROP_CONCEPT", columnList = "CONCEPT_PID", unique = false)
})
public class TermConceptProperty implements Serializable {
public static final int MAX_PROPTYPE_ENUM_LENGTH = 6;

46
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -41,16 +41,26 @@ import ca.uhn.fhir.jpa.model.entity.SearchParamPresent;
import ca.uhn.fhir.util.VersionEnum;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.util.stream.Collectors.toSet;
@SuppressWarnings({"SqlNoDataSourceInspection", "SpellCheckingInspection"})
public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
private final Set<FlagEnum> myFlags;
// H2, Derby, MariaDB, and MySql automatically add indexes to foreign keys
public static final DriverTypeEnum[] NON_AUTOMATIC_FK_INDEX_PLATFORMS = new DriverTypeEnum[] {
DriverTypeEnum.POSTGRES_9_4, DriverTypeEnum.ORACLE_12C, DriverTypeEnum.MSSQL_2012 };
/**
* Constructor
*/
@ -76,8 +86,44 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
init540(); // 20210218 - 20210520
init550(); // 20210520 -
init560(); // 20211027 -
init570(); // 20211102 -
}
private void init570() {
Builder version = forVersion(VersionEnum.V5_7_0);
// both indexes must have same name that indexed FK or SchemaMigrationTest complains because H2 sets this index automatically
version.onTable("TRM_CONCEPT_PROPERTY")
.addIndex("20211102.1", "FK_CONCEPTPROP_CONCEPT")
.unique(false)
.withColumns("CONCEPT_PID")
.onlyAppliesToPlatforms(NON_AUTOMATIC_FK_INDEX_PLATFORMS);
version.onTable("TRM_CONCEPT_DESIG")
.addIndex("20211102.2", "FK_CONCEPTDESIG_CONCEPT")
.unique(false)
.withColumns("CONCEPT_PID")
// H2, Derby, MariaDB, and MySql automatically add indexes to foreign keys
.onlyAppliesToPlatforms(NON_AUTOMATIC_FK_INDEX_PLATFORMS);
version.onTable("TRM_CONCEPT_PC_LINK")
.addIndex("20211102.3", "FK_TERM_CONCEPTPC_CHILD")
.unique(false)
.withColumns("CHILD_PID")
// H2, Derby, MariaDB, and MySql automatically add indexes to foreign keys
.onlyAppliesToPlatforms(NON_AUTOMATIC_FK_INDEX_PLATFORMS);
version.onTable("TRM_CONCEPT_PC_LINK")
.addIndex("20211102.4", "FK_TERM_CONCEPTPC_PARENT")
.unique(false)
.withColumns("PARENT_PID")
// H2, Derby, MariaDB, and MySql automatically add indexes to foreign keys
.onlyAppliesToPlatforms(NON_AUTOMATIC_FK_INDEX_PLATFORMS);
}
private void init560() {
init560_20211027();
}

72
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java
View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.interceptor.model.RequestPartitionId;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
@ -59,8 +60,9 @@ import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ValueSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronizationManager;
@ -86,6 +88,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
@ -108,8 +111,6 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
@Autowired
protected IdHelperService myIdHelperService;
@Autowired
private PlatformTransactionManager myTransactionManager;
@Autowired
private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
@Autowired
private ITermVersionAdapterSvc myTerminologyVersionAdapterSvc;
@ -124,6 +125,13 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
@Autowired
private IResourceTableDao myResourceTableDao;
@Autowired
private IBatchJobSubmitter myJobSubmitter;
@Autowired @Qualifier(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME)
private Job myTermCodeSystemVersionDeleteJob;
@Override
public ResourcePersistentId getValueSetResourcePid(IIdType theIdType) {
return myIdHelperService.resolveResourcePersistentIds(RequestPartitionId.allPartitions(), theIdType.getResourceType(), theIdType.getIdPart());
@ -262,44 +270,6 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
return childTermConcepts;
}
@Override
@Transactional
public void deleteCodeSystem(TermCodeSystem theCodeSystem) {
assert TransactionSynchronizationManager.isActualTransactionActive();
ourLog.info(" * Deleting code system {}", theCodeSystem.getPid());
myEntityManager.flush();
TermCodeSystem cs = myCodeSystemDao.findById(theCodeSystem.getPid()).orElseThrow(IllegalStateException::new);
cs.setCurrentVersion(null);
myCodeSystemDao.save(cs);
myCodeSystemDao.flush();
List<TermCodeSystemVersion> codeSystemVersions = myCodeSystemVersionDao.findByCodeSystemPid(theCodeSystem.getPid());
List<Long> codeSystemVersionPids = codeSystemVersions
.stream()
.map(TermCodeSystemVersion::getPid)
.collect(Collectors.toList());
for (Long next : codeSystemVersionPids) {
deleteCodeSystemVersion(next);
}
myCodeSystemVersionDao.deleteForCodeSystem(theCodeSystem);
myCodeSystemDao.delete(theCodeSystem);
myEntityManager.flush();
}
@Override
@Transactional(propagation = Propagation.NEVER)
public void deleteCodeSystemVersion(TermCodeSystemVersion theCodeSystemVersion) {
assert !TransactionSynchronizationManager.isActualTransactionActive();
// Delete TermCodeSystemVersion
ourLog.info(" * Deleting TermCodeSystemVersion {}", theCodeSystemVersion.getCodeSystemVersionId());
deleteCodeSystemVersion(theCodeSystemVersion.getPid());
}
/**
* Returns the number of saved concepts
*/
@ -512,26 +482,6 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
return existing;
}
private void deleteCodeSystemVersion(final Long theCodeSystemVersionPid) {
assert TransactionSynchronizationManager.isActualTransactionActive();
ourLog.info(" * Marking code system version {} for deletion", theCodeSystemVersionPid);
Optional<TermCodeSystem> codeSystemOpt = myCodeSystemDao.findWithCodeSystemVersionAsCurrentVersion(theCodeSystemVersionPid);
if (codeSystemOpt.isPresent()) {
TermCodeSystem codeSystem = codeSystemOpt.get();
if (codeSystem.getCurrentVersion() != null && codeSystem.getCurrentVersion().getPid().equals(theCodeSystemVersionPid)) {
ourLog.info(" * Removing code system version {} as current version of code system {}", theCodeSystemVersionPid, codeSystem.getPid());
codeSystem.setCurrentVersion(null);
myCodeSystemDao.save(codeSystem);
}
}
TermCodeSystemVersion codeSystemVersion = myCodeSystemVersionDao.findById(theCodeSystemVersionPid).orElseThrow(() -> new IllegalStateException());
codeSystemVersion.setCodeSystemVersionId("DELETED_" + UUID.randomUUID().toString());
myCodeSystemVersionDao.save(codeSystemVersion);
myDeferredStorageSvc.deleteCodeSystemVersion(codeSystemVersion);
}
private void validateDstu3OrNewer() {
Validate.isTrue(myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3), "Terminology operations only supported in DSTU3+ mode");

187
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View File

@ -20,12 +20,11 @@ package ca.uhn.fhir.jpa.term;
* #L%
*/
import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
@ -37,6 +36,7 @@ import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.util.StopWatch;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.Validate;
@ -45,10 +45,15 @@ import org.hl7.fhir.r4.model.ValueSet;
import org.quartz.JobExecutionContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.launch.JobExecutionNotRunningException;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
@ -59,13 +64,17 @@ import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_VERSION_ID;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
private static final Logger ourLog = LoggerFactory.getLogger(TermDeferredStorageSvcImpl.class);
@ -75,6 +84,9 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
final private List<ValueSet> myDeferredValueSets = Collections.synchronizedList(new ArrayList<>());
final private List<ConceptMap> myDeferredConceptMaps = Collections.synchronizedList(new ArrayList<>());
final private List<TermConceptParentChildLink> myConceptLinksToSaveLater = Collections.synchronizedList(new ArrayList<>());
final private List<JobExecution> myCurrentJobExecutions = Collections.synchronizedList(new ArrayList<>());
@Autowired
protected ITermConceptDao myConceptDao;
@Autowired
@ -83,10 +95,6 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
protected ITermCodeSystemVersionDao myCodeSystemVersionDao;
@Autowired
protected PlatformTransactionManager myTransactionMgr;
@Autowired
protected ITermConceptPropertyDao myConceptPropertyDao;
@Autowired
protected ITermConceptDesignationDao myConceptDesignationDao;
private boolean myProcessDeferred = true;
@Autowired
private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
@ -97,6 +105,19 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
@Autowired
private ITermCodeSystemStorageSvc myCodeSystemStorageSvc;
@Autowired
private IBatchJobSubmitter myJobSubmitter;
@Autowired
private JobOperator myJobOperator;
@Autowired @Qualifier(TERM_CODE_SYSTEM_DELETE_JOB_NAME)
private org.springframework.batch.core.Job myTermCodeSystemDeleteJob;
@Autowired @Qualifier(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME)
private org.springframework.batch.core.Job myTermCodeSystemVersionDeleteJob;
@Override
public void addConceptToStorageQueue(TermConcept theConcept) {
Validate.notNull(theConcept);
@ -122,28 +143,26 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
}
@Override
@Transactional
public void deleteCodeSystem(TermCodeSystem theCodeSystem) {
theCodeSystem.setCodeSystemUri("urn:uuid:" + UUID.randomUUID().toString());
myCodeSystemDao.save(theCodeSystem);
myDeferredCodeSystemsDeletions.add(theCodeSystem);
}
@Override
@Transactional
public void deleteCodeSystemForResource(ResourceTable theCodeSystemToDelete) {
// there are use cases (at least in tests) where the code system is not present for the resource but versions are,
// so, as code system deletion also deletes versions, we try the system first but if not present we also try versions
TermCodeSystem termCodeSystemToDelete = myCodeSystemDao.findByResourcePid(theCodeSystemToDelete.getResourceId());
if (termCodeSystemToDelete != null) {
termCodeSystemToDelete.setCodeSystemUri("urn:uuid:" + UUID.randomUUID());
myCodeSystemDao.save(termCodeSystemToDelete);
myDeferredCodeSystemsDeletions.add(termCodeSystemToDelete);
return;
}
List<TermCodeSystemVersion> codeSystemVersionsToDelete = myCodeSystemVersionDao.findByCodeSystemResourcePid(theCodeSystemToDelete.getResourceId());
for (TermCodeSystemVersion codeSystemVersionToDelete : codeSystemVersionsToDelete) {
if (codeSystemVersionToDelete != null) {
myDeferredCodeSystemVersionsDeletions.add(codeSystemVersionToDelete);
}
}
TermCodeSystem codeSystemToDelete = myCodeSystemDao.findByResourcePid(theCodeSystemToDelete.getResourceId());
if (codeSystemToDelete != null) {
deleteCodeSystem(codeSystemToDelete);
}
}
@Override
public void setProcessDeferred(boolean theProcessDeferred) {
myProcessDeferred = theProcessDeferred;
@ -236,14 +255,26 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
myDeferredCodeSystemsDeletions.clear();
myConceptLinksToSaveLater.clear();
myDeferredCodeSystemVersionsDeletions.clear();
clearJobExecutions();
}
private void runInTransaction(Runnable theRunnable) {
assert !TransactionSynchronizationManager.isActualTransactionActive();
new TransactionTemplate(myTransactionMgr).executeWithoutResult(tx -> theRunnable.run());
private void clearJobExecutions() {
for (JobExecution jobExecution : myCurrentJobExecutions) {
if (! jobExecution.isRunning()) { continue; }
try {
myJobOperator.stop(jobExecution.getId());
} catch (NoSuchJobExecutionException | JobExecutionNotRunningException theE) {
ourLog.error("Couldn't stop job execution {}: {}", jobExecution.getId(), theE);
}
}
myCurrentJobExecutions.clear();
}
private <T> T runInTransaction(Supplier<T> theRunnable) {
assert !TransactionSynchronizationManager.isActualTransactionActive();
@ -315,98 +346,53 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
return !myDeferredCodeSystemVersionsDeletions.isEmpty();
}
private void processDeferredCodeSystemDeletions() {
for (TermCodeSystem next : myDeferredCodeSystemsDeletions) {
myCodeSystemStorageSvc.deleteCodeSystem(next);
deleteTermCodeSystemOffline(next.getPid());
}
myDeferredCodeSystemsDeletions.clear();
}
private void processDeferredCodeSystemVersionDeletions() {
for (TermCodeSystemVersion next : myDeferredCodeSystemVersionsDeletions) {
processDeferredCodeSystemVersionDeletions(next.getPid());
deleteTermCodeSystemVersionOffline(next.getPid());
}
myDeferredCodeSystemVersionsDeletions.clear();
}
private void processDeferredCodeSystemVersionDeletions(long theCodeSystemVersionPid) {
assert !TransactionSynchronizationManager.isActualTransactionActive();
ourLog.info(" * Deleting CodeSystemVersion[id={}]", theCodeSystemVersionPid);
PageRequest page1000 = PageRequest.of(0, 1000);
private void deleteTermCodeSystemVersionOffline(Long theCodeSystemVersionPid) {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_VERSION_ID, new JobParameter(theCodeSystemVersionPid, true) ));
// Parent/Child links
{
String descriptor = "parent/child links";
Supplier<Slice<Long>> loader = () -> myConceptParentChildLinkDao.findIdsByCodeSystemVersion(page1000, theCodeSystemVersionPid);
Supplier<Integer> counter = () -> myConceptParentChildLinkDao.countByCodeSystemVersion(theCodeSystemVersionPid);
doDelete(descriptor, loader, counter, myConceptParentChildLinkDao);
try {
JobExecution jobExecution = myJobSubmitter.runJob(myTermCodeSystemVersionDeleteJob, jobParameters);
myCurrentJobExecutions.add(jobExecution);
} catch (JobParametersInvalidException theE) {
throw new InternalErrorException("Offline job submission for TermCodeSystemVersion: " +
theCodeSystemVersionPid + " failed: " + theE);
}
// Properties
{
String descriptor = "concept properties";
Supplier<Slice<Long>> loader = () -> myConceptPropertyDao.findIdsByCodeSystemVersion(page1000, theCodeSystemVersionPid);
Supplier<Integer> counter = () -> myConceptPropertyDao.countByCodeSystemVersion(theCodeSystemVersionPid);
doDelete(descriptor, loader, counter, myConceptPropertyDao);
}
// Designations
{
String descriptor = "concept designations";
Supplier<Slice<Long>> loader = () -> myConceptDesignationDao.findIdsByCodeSystemVersion(page1000, theCodeSystemVersionPid);
Supplier<Integer> counter = () -> myConceptDesignationDao.countByCodeSystemVersion(theCodeSystemVersionPid);
doDelete(descriptor, loader, counter, myConceptDesignationDao);
}
// Concepts
{
String descriptor = "concepts";
// For some reason, concepts are much slower to delete, so use a smaller batch size
PageRequest page100 = PageRequest.of(0, 100);
Supplier<Slice<Long>> loader = () -> myConceptDao.findIdsByCodeSystemVersion(page100, theCodeSystemVersionPid);
Supplier<Integer> counter = () -> myConceptDao.countByCodeSystemVersion(theCodeSystemVersionPid);
doDelete(descriptor, loader, counter, myConceptDao);
}
runInTransaction(() -> {
Optional<TermCodeSystem> codeSystemOpt = myCodeSystemDao.findWithCodeSystemVersionAsCurrentVersion(theCodeSystemVersionPid);
if (codeSystemOpt.isPresent()) {
TermCodeSystem codeSystem = codeSystemOpt.get();
ourLog.info(" * Removing code system version {} as current version of code system {}", theCodeSystemVersionPid, codeSystem.getPid());
codeSystem.setCurrentVersion(null);
myCodeSystemDao.save(codeSystem);
}
ourLog.info(" * Deleting code system version");
Optional<TermCodeSystemVersion> csv = myCodeSystemVersionDao.findById(theCodeSystemVersionPid);
if (csv.isPresent()) {
myCodeSystemVersionDao.delete(csv.get());
}
});
}
private <T> void doDelete(String theDescriptor, Supplier<Slice<Long>> theLoader, Supplier<Integer> theCounter, JpaRepository<T, Long> theDao) {
assert !TransactionSynchronizationManager.isActualTransactionActive();
int count;
ourLog.info(" * Deleting {}", theDescriptor);
int totalCount = runInTransaction(theCounter);
StopWatch sw = new StopWatch();
count = 0;
while (true) {
Slice<Long> link = runInTransaction(theLoader);
if (!link.hasContent()) {
break;
}
private void deleteTermCodeSystemOffline(Long theCodeSystemPid) {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_ID, new JobParameter(theCodeSystemPid, true) ));
runInTransaction(() -> link.forEach(theDao::deleteById));
try {
count += link.getNumberOfElements();
ourLog.info(" * {} {} deleted ({}/{}) remaining - {}/sec - ETA: {}", count, theDescriptor, count, totalCount, sw.formatThroughput(count, TimeUnit.SECONDS), sw.getEstimatedTimeRemaining(count, totalCount));
JobExecution jobExecution = myJobSubmitter.runJob(myTermCodeSystemDeleteJob, jobParameters);
myCurrentJobExecutions.add(jobExecution);
} catch (JobParametersInvalidException theE) {
throw new InternalErrorException("Offline job submission for TermCodeSystem: " +
theCodeSystemPid + " failed: " + theE);
}
}
@ -419,9 +405,14 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
retVal &= !isDeferredValueSets();
retVal &= !isDeferredConceptMaps();
retVal &= !isDeferredCodeSystemDeletions();
retVal &= !isJobsExecuting();
return retVal;
}
private boolean isJobsExecuting() {
return myCurrentJobExecutions.stream().anyMatch(JobExecution::isRunning);
}
private void saveConceptLink(TermConceptParentChildLink next) {
if (next.getId() == null) {

5
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/api/ITermCodeSystemStorageSvc.java
View File

@ -51,11 +51,6 @@ public interface ITermCodeSystemStorageSvc {
(boolean) theRequestDetails.getUserData().getOrDefault(MAKE_LOADING_VERSION_CURRENT, Boolean.TRUE);
}
void deleteCodeSystem(TermCodeSystem theCodeSystem);
void deleteCodeSystemVersion(TermCodeSystemVersion theCodeSystemVersion);
void storeNewCodeSystemVersion(ResourcePersistentId theCodeSystemResourcePid, String theSystemUri, String theSystemName,
String theSystemVersionId, TermCodeSystemVersion theCodeSystemVersion, ResourceTable theCodeSystemResourceTable,
RequestDetails theRequestDetails);

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/api/ITermDeferredStorageSvc.java
View File

@ -54,8 +54,6 @@ public interface ITermDeferredStorageSvc {
void addValueSetsToStorageQueue(List<ValueSet> theValueSets);
void deleteCodeSystem(TermCodeSystem theCodeSystem);
void deleteCodeSystemForResource(ResourceTable theCodeSystemResourceToDelete);
void deleteCodeSystemVersion(TermCodeSystemVersion theCodeSystemVersion);

62
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BaseTermCodeSystemDeleteJobConfig.java Normal file
View File

@ -0,0 +1,62 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Configuration artifacts common to TermCodeSystemDeleteJobConfig and TermCodeSystemVersionDeleteJobConfig
**/
@Configuration
public class BaseTermCodeSystemDeleteJobConfig {
protected static final int TERM_CONCEPT_DELETE_TIMEOUT = 60 * 2; // two minutes
@Autowired
protected JobBuilderFactory myJobBuilderFactory;
@Autowired
protected StepBuilderFactory myStepBuilderFactory;
@Bean
public BatchTermCodeSystemVersionDeleteWriter batchTermCodeSystemVersionDeleteWriter() {
return new BatchTermCodeSystemVersionDeleteWriter();
}
@Bean
public BatchConceptRelationsDeleteWriter batchConceptRelationsDeleteWriter() {
return new BatchConceptRelationsDeleteWriter();
}
@Bean
public BatchTermConceptsDeleteWriter batchTermConceptsDeleteWriter() {
return new BatchTermConceptsDeleteWriter();
}
}

66
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BatchConceptRelationsDeleteWriter.java Normal file
View File

@ -0,0 +1,66 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import java.text.DecimalFormat;
import java.util.List;
public class BatchConceptRelationsDeleteWriter implements ItemWriter<Long> {
private static final Logger ourLog = LoggerFactory.getLogger(BatchConceptRelationsDeleteWriter.class);
private static final DecimalFormat ourDecimalFormat = new DecimalFormat("#,###");
@Autowired
private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
@Autowired
private ITermConceptPropertyDao myConceptPropertyDao;
@Autowired
private ITermConceptDesignationDao myConceptDesignationDao;
@Override
public void write(List<? extends Long> theTermCodeSystemVersionPidList) throws Exception {
// receives input in chunks of size one
long codeSystemVersionId = theTermCodeSystemVersionPidList.get(0);
ourLog.info("Deleting term code links");
int deletedLinks = myConceptParentChildLinkDao.deleteByCodeSystemVersion(codeSystemVersionId);
ourLog.info("Deleted {} term code links", ourDecimalFormat.format(deletedLinks));
ourLog.info("Deleting term code properties");
int deletedProperties = myConceptPropertyDao.deleteByCodeSystemVersion(codeSystemVersionId);
ourLog.info("Deleted {} term code properties", ourDecimalFormat.format(deletedProperties));
ourLog.info("Deleting concept designations");
int deletedDesignations = myConceptDesignationDao.deleteByCodeSystemVersion(codeSystemVersionId);
ourLog.info("Deleted {} concept designations", ourDecimalFormat.format(deletedDesignations));
}
}

54
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BatchTermCodeSystemUniqueVersionDeleteReader.java Normal file
View File

@ -0,0 +1,54 @@
package ca.uhn.fhir.jpa.term.job;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Value;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_VERSION_ID;
/**
* This reader works as a pass-through by passing the received parameter once to the writer,
* in order to share the writer functionality between two jobs
*/
public class BatchTermCodeSystemUniqueVersionDeleteReader implements ItemReader<Long> {
private static final Logger ourLog = LoggerFactory.getLogger(BatchTermCodeSystemUniqueVersionDeleteReader.class);
@Value("#{jobParameters['" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "']}")
private Long myTermCodeSystemVersionPid;
// indicates if the parameter was already passed once to the writer, which indicates end of task
private boolean myParameterPassed;
@Override
public Long read() throws Exception {
if ( ! myParameterPassed) {
myParameterPassed = true;
return myTermCodeSystemVersionPid;
}
return null;
}
}

75
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BatchTermCodeSystemVersionDeleteReader.java Normal file
View File

@ -0,0 +1,75 @@
package ca.uhn.fhir.jpa.term.job;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.util.List;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
/**
*
*/
public class BatchTermCodeSystemVersionDeleteReader implements ItemReader<Long> {
private static final Logger ourLog = LoggerFactory.getLogger(BatchTermCodeSystemVersionDeleteReader.class);
@Autowired
private ITermCodeSystemVersionDao myTermCodeSystemVersionDao;
@Value("#{jobParameters['" + JOB_PARAM_CODE_SYSTEM_ID + "']}")
private Long myTermCodeSystemPid;
private List<Long> myTermCodeSystemVersionPidList;
private int myCurrentIdx = 0;
@Override
public Long read() throws Exception {
if (myTermCodeSystemVersionPidList == null) {
myTermCodeSystemVersionPidList = myTermCodeSystemVersionDao.findSortedPidsByCodeSystemPid(myTermCodeSystemPid);
}
if (myTermCodeSystemVersionPidList.isEmpty()) {
// nothing to process
ourLog.info("Nothing to process");
return null;
}
if (myCurrentIdx >= myTermCodeSystemVersionPidList.size()) {
// nothing else to process
ourLog.info("No more versions to process");
return null;
}
// still processing elements
long TermCodeSystemVersionPid = myTermCodeSystemVersionPidList.get(myCurrentIdx++);
ourLog.info("Passing termCodeSystemVersionPid: {} to writer", TermCodeSystemVersionPid);
return TermCodeSystemVersionPid;
}
}

77
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BatchTermCodeSystemVersionDeleteWriter.java Normal file
View File

@ -0,0 +1,77 @@
package ca.uhn.fhir.jpa.term.job;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.text.DecimalFormat;
import java.util.List;
import java.util.Optional;
@Component
public class BatchTermCodeSystemVersionDeleteWriter implements ItemWriter<Long> {
private static final Logger ourLog = LoggerFactory.getLogger(BatchTermCodeSystemVersionDeleteWriter.class);
@Autowired
private ITermCodeSystemDao myCodeSystemDao;
@Autowired
private ITermCodeSystemVersionDao myTermCodeSystemVersionDao;
@Override
public void write(List<? extends Long> theTermCodeSystemVersionPidList) throws Exception {
// receives input in chunks of size one
long codeSystemVersionId = theTermCodeSystemVersionPidList.get(0);
ourLog.debug("Executing for codeSystemVersionId: {}", codeSystemVersionId);
// if TermCodeSystemVersion being deleted is current, disconnect it form TermCodeSystem
Optional<TermCodeSystem> codeSystemOpt = myCodeSystemDao.findWithCodeSystemVersionAsCurrentVersion(codeSystemVersionId);
if (codeSystemOpt.isPresent()) {
TermCodeSystem codeSystem = codeSystemOpt.get();
ourLog.info("Removing code system version: {} as current version of code system: {}", codeSystemVersionId, codeSystem.getPid());
codeSystem.setCurrentVersion(null);
myCodeSystemDao.save(codeSystem);
}
ourLog.info("Deleting code system version: {}", codeSystemVersionId);
Optional<TermCodeSystemVersion> csv = myTermCodeSystemVersionDao.findById(codeSystemVersionId);
csv.ifPresent(theTermCodeSystemVersion -> {
myTermCodeSystemVersionDao.delete(theTermCodeSystemVersion);
ourLog.info("Code system version: {} deleted", codeSystemVersionId);
});
}
}

51
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/BatchTermConceptsDeleteWriter.java Normal file
View File

@ -0,0 +1,51 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import java.text.DecimalFormat;
import java.util.List;
public class BatchTermConceptsDeleteWriter implements ItemWriter<Long> {
private static final Logger ourLog = LoggerFactory.getLogger(BatchTermConceptsDeleteWriter.class);
private static final DecimalFormat ourDecimalFormat = new DecimalFormat("#,###");
@Autowired
private ITermConceptDao myConceptDao;
@Override
public void write(List<? extends Long> theTermCodeSystemVersionPidList) throws Exception {
// receives input in chunks of size one
long codeSystemVersionId = theTermCodeSystemVersionPidList.get(0);
ourLog.info("Deleting concepts");
int deletedConcepts = myConceptDao.deleteByCodeSystemVersion(codeSystemVersionId);
ourLog.info("Deleted {} concepts", ourDecimalFormat.format(deletedConcepts));
}
}

122
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobConfig.java Normal file
View File

@ -0,0 +1,122 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParametersValidator;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.transaction.interceptor.DefaultTransactionAttribute;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_STEP_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_STEP_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CONCEPTS_DELETE_STEP_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CONCEPT_RELATIONS_DELETE_STEP_NAME;
/**
* Configuration for batch job which deletes a TermCodeSystem and its related TermCodeSystemVersion(s),
* TermConceptProperty(es), TermConceptDesignation(s), and TermConceptParentChildLink(s)
**/
@Configuration
public class TermCodeSystemDeleteJobConfig extends BaseTermCodeSystemDeleteJobConfig {
@Bean(name = TERM_CODE_SYSTEM_DELETE_JOB_NAME)
@Lazy
public Job termCodeSystemDeleteJob() {
return myJobBuilderFactory.get(TERM_CODE_SYSTEM_DELETE_JOB_NAME)
.validator(termCodeSystemDeleteJobParameterValidator())
.start(termConceptRelationsDeleteStep())
.next(termConceptsDeleteStep())
.next(termCodeSystemVersionDeleteStep())
.next(termCodeSystemDeleteStep())
.build();
}
@Bean
public JobParametersValidator termCodeSystemDeleteJobParameterValidator() {
return new TermCodeSystemDeleteJobParameterValidator();
}
/**
* This steps deletes TermConceptParentChildLink(s), TermConceptProperty(es) and TermConceptDesignation(s)
* related to TermConcept(s) of the TermCodeSystemVersion being deleted
*/
@Bean(name = TERM_CONCEPT_RELATIONS_DELETE_STEP_NAME)
public Step termConceptRelationsDeleteStep() {
return myStepBuilderFactory.get(TERM_CONCEPT_RELATIONS_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemVersionDeleteReader())
.writer(batchConceptRelationsDeleteWriter())
.build();
}
/**
* This steps deletes TermConcept(s) of the TermCodeSystemVersion being deleted
*/
@Bean(name = TERM_CONCEPTS_DELETE_STEP_NAME)
public Step termConceptsDeleteStep() {
DefaultTransactionAttribute attribute = new DefaultTransactionAttribute();
attribute.setTimeout(TERM_CONCEPT_DELETE_TIMEOUT);
return myStepBuilderFactory.get(TERM_CONCEPTS_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemVersionDeleteReader())
.writer(batchTermConceptsDeleteWriter())
.transactionAttribute(attribute)
.build();
}
/**
* This steps deletes the TermCodeSystemVersion
*/
@Bean(name = TERM_CODE_SYSTEM_VERSION_DELETE_STEP_NAME)
public Step termCodeSystemVersionDeleteStep() {
return myStepBuilderFactory.get(TERM_CODE_SYSTEM_VERSION_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemVersionDeleteReader())
.writer(batchTermCodeSystemVersionDeleteWriter())
.build();
}
@Bean(name = TERM_CODE_SYSTEM_DELETE_STEP_NAME)
public Step termCodeSystemDeleteStep() {
return myStepBuilderFactory.get(TERM_CODE_SYSTEM_DELETE_STEP_NAME)
.tasklet(termCodeSystemDeleteTasklet())
.build();
}
@Bean
@StepScope
public BatchTermCodeSystemVersionDeleteReader batchTermCodeSystemVersionDeleteReader() {
return new BatchTermCodeSystemVersionDeleteReader();
}
@Bean
public TermCodeSystemDeleteTasklet termCodeSystemDeleteTasklet() {
return new TermCodeSystemDeleteTasklet();
}
}

53
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobParameterValidator.java Normal file
View File

@ -0,0 +1,53 @@
package ca.uhn.fhir.jpa.term.job;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.JobParametersValidator;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
/**
* Validates that a TermCodeSystem parameter is present
*/
public class TermCodeSystemDeleteJobParameterValidator implements JobParametersValidator {
@Override
public void validate(JobParameters theJobParameters) throws JobParametersInvalidException {
if (theJobParameters == null) {
throw new JobParametersInvalidException("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_ID + "'");
}
if ( ! theJobParameters.getParameters().containsKey(JOB_PARAM_CODE_SYSTEM_ID)) {
throw new JobParametersInvalidException("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_ID + "'");
}
Long termCodeSystemPid = theJobParameters.getLong(JOB_PARAM_CODE_SYSTEM_ID);
if (termCodeSystemPid == null) {
throw new JobParametersInvalidException("'" + JOB_PARAM_CODE_SYSTEM_ID + "' parameter is null");
}
if (termCodeSystemPid <= 0) {
throw new JobParametersInvalidException("Invalid parameter '" + JOB_PARAM_CODE_SYSTEM_ID + "' value: " + termCodeSystemPid);
}
}
}

60
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteTasklet.java Normal file
View File

@ -0,0 +1,60 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
@Component
public class TermCodeSystemDeleteTasklet implements Tasklet {
private static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemDeleteTasklet.class);
@Autowired
private ITermCodeSystemDao myTermCodeSystemDao;
@Autowired
private ITermCodeSystemVersionDao myCodeSystemVersionDao;
@Override
public RepeatStatus execute(@NotNull StepContribution contribution, ChunkContext context) throws Exception {
long codeSystemPid = (Long) context.getStepContext().getJobParameters().get(JOB_PARAM_CODE_SYSTEM_ID);
ourLog.info("Deleting code system {}", codeSystemPid);
myTermCodeSystemDao.findById(codeSystemPid).orElseThrow(IllegalStateException::new);
myTermCodeSystemDao.deleteById(codeSystemPid);
ourLog.info("Code system {} deleted", codeSystemPid);
return RepeatStatus.FINISHED;
}
}

105
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemVersionDeleteJobConfig.java Normal file
View File

@ -0,0 +1,105 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParametersValidator;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.transaction.interceptor.DefaultTransactionAttribute;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_UNIQUE_VERSION_DELETE_STEP_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CONCEPTS_UNIQUE_VERSION_DELETE_STEP_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CONCEPT_RELATIONS_UNIQUE_VERSION_DELETE_STEP_NAME;
/**
* Configuration for batch job which deletes a specific TermCodeSystemVersion and its related,
* TermConceptProperty(es), TermConceptDesignation(s), and TermConceptParentChildLink(s)
**/
@Configuration
public class TermCodeSystemVersionDeleteJobConfig extends BaseTermCodeSystemDeleteJobConfig {
@Bean(name = TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME)
@Lazy
public Job termCodeSystemVersionDeleteJob() {
return myJobBuilderFactory.get(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME)
.validator(termCodeSystemVersionDeleteJobParameterValidator())
.start(termConceptRelationsUniqueVersionDeleteStep())
.next(termConceptsUniqueVersionDeleteStep())
.next(termCodeSystemUniqueVersionDeleteStep())
.build();
}
@Bean
public JobParametersValidator termCodeSystemVersionDeleteJobParameterValidator() {
return new TermCodeSystemVersionDeleteJobParameterValidator();
}
@Bean(name = TERM_CONCEPT_RELATIONS_UNIQUE_VERSION_DELETE_STEP_NAME)
public Step termConceptRelationsUniqueVersionDeleteStep() {
return myStepBuilderFactory.get(TERM_CONCEPT_RELATIONS_UNIQUE_VERSION_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemUniqueVersionDeleteReader())
.writer(batchConceptRelationsDeleteWriter())
.build();
}
@Bean(name = TERM_CONCEPTS_UNIQUE_VERSION_DELETE_STEP_NAME)
public Step termConceptsUniqueVersionDeleteStep() {
DefaultTransactionAttribute attribute = new DefaultTransactionAttribute();
attribute.setTimeout(TERM_CONCEPT_DELETE_TIMEOUT);
return myStepBuilderFactory.get(TERM_CONCEPTS_UNIQUE_VERSION_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemUniqueVersionDeleteReader())
.writer(batchTermConceptsDeleteWriter())
.transactionAttribute(attribute)
.build();
}
@Bean(name = TERM_CODE_SYSTEM_UNIQUE_VERSION_DELETE_STEP_NAME)
public Step termCodeSystemUniqueVersionDeleteStep() {
return myStepBuilderFactory.get(TERM_CODE_SYSTEM_UNIQUE_VERSION_DELETE_STEP_NAME)
.<Long, Long>chunk(1)
.reader(batchTermCodeSystemUniqueVersionDeleteReader())
.writer(batchTermCodeSystemVersionDeleteWriter())
.build();
}
@Bean
@StepScope
public BatchTermCodeSystemUniqueVersionDeleteReader batchTermCodeSystemUniqueVersionDeleteReader() {
return new BatchTermCodeSystemUniqueVersionDeleteReader();
}
}

54
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemVersionDeleteJobParameterValidator.java Normal file
View File

@ -0,0 +1,54 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.JobParametersValidator;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_VERSION_ID;
/**
* Validates that a TermCodeSystem parameter is present
*/
public class TermCodeSystemVersionDeleteJobParameterValidator implements JobParametersValidator {
@Override
public void validate(JobParameters theJobParameters) throws JobParametersInvalidException {
if (theJobParameters == null) {
throw new JobParametersInvalidException("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "'");
}
if ( ! theJobParameters.getParameters().containsKey(JOB_PARAM_CODE_SYSTEM_VERSION_ID)) {
throw new JobParametersInvalidException("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "'");
}
Long termCodeSystemPid = theJobParameters.getLong(JOB_PARAM_CODE_SYSTEM_VERSION_ID);
if (termCodeSystemPid == null) {
throw new JobParametersInvalidException("'" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "' parameter is null");
}
if (termCodeSystemPid <= 0) {
throw new JobParametersInvalidException(
"Invalid parameter '" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "' value: " + termCodeSystemPid);
}
}
}

62
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/job/TermConceptDeleteTasklet.java Normal file
View File

@ -0,0 +1,62 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
/**
* Deletes the TermConcept(s) related to the TermCodeSystemVersion being deleted
* Executes in its own step to be in own transaction because it is a DB-heavy operation
*/
@Component
public class TermConceptDeleteTasklet implements Tasklet {
private static final Logger ourLog = LoggerFactory.getLogger(TermConceptDeleteTasklet.class);
@Autowired
private ITermCodeSystemDao myTermCodeSystemDao;
@Autowired
private ITermCodeSystemVersionDao myCodeSystemVersionDao;
@Override
public RepeatStatus execute(@NotNull StepContribution contribution, ChunkContext context) throws Exception {
long codeSystemPid = (Long) context.getStepContext().getJobParameters().get(JOB_PARAM_CODE_SYSTEM_ID);
ourLog.info("Deleting code system {}", codeSystemPid);
myTermCodeSystemDao.findById(codeSystemPid).orElseThrow(IllegalStateException::new);
myTermCodeSystemDao.deleteById(codeSystemPid);
return RepeatStatus.FINISHED;
}
}

10
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3CodeSystemTest.java
View File

@ -1,21 +1,28 @@
package ca.uhn.fhir.jpa.dao.dstu3;
import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.apache.commons.io.IOUtils;
import org.hl7.fhir.dstu3.model.CodeSystem;
import org.hl7.fhir.dstu3.model.Enumerations;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
public class FhirResourceDaoDstu3CodeSystemTest extends BaseJpaDstu3Test {
@Autowired private BatchJobHelper myBatchJobHelper;
@AfterAll
public static void afterClassClearContext() {
TermReindexingSvcImpl.setForceSaveDeferredAlwaysForUnitTest(false);
@ -64,6 +71,7 @@ public class FhirResourceDaoDstu3CodeSystemTest extends BaseJpaDstu3Test {
cs.addConcept().setCode("B");
myCodeSystemDao.update(cs, mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runInTransaction(()->{
assertEquals(2, myConceptDao.count());
});
@ -77,6 +85,7 @@ public class FhirResourceDaoDstu3CodeSystemTest extends BaseJpaDstu3Test {
cs.addConcept().setCode("C");
myCodeSystemDao.update(cs, mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runInTransaction(()->{
assertEquals(1, myConceptDao.count());
});
@ -86,6 +95,7 @@ public class FhirResourceDaoDstu3CodeSystemTest extends BaseJpaDstu3Test {
myCodeSystemDao.delete(id);
});
myTerminologyDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
runInTransaction(()->{
assertEquals(0L, myConceptDao.count());
});

9
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4CodeSystemTest.java
View File

@ -2,15 +2,19 @@ package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.apache.commons.io.IOUtils;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -18,6 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertNull;
public class FhirResourceDaoR4CodeSystemTest extends BaseJpaR4Test {
@Autowired private BatchJobHelper myBatchJobHelper;
@Test
public void testIndexContained() throws Exception {
TermReindexingSvcImpl.setForceSaveDeferredAlwaysForUnitTest(true);
@ -58,6 +64,7 @@ public class FhirResourceDaoR4CodeSystemTest extends BaseJpaR4Test {
// Now the background scheduler will do its thing
myTerminologyDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
runInTransaction(() -> {
assertEquals(0, myTermCodeSystemDao.count());
assertEquals(0, myTermCodeSystemVersionDao.count());
@ -116,6 +123,7 @@ public class FhirResourceDaoR4CodeSystemTest extends BaseJpaR4Test {
// Now the background scheduler will do its thing
myTerminologyDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
// Entities for first resource should be gone now.
runInTransaction(() -> {
@ -150,6 +158,7 @@ public class FhirResourceDaoR4CodeSystemTest extends BaseJpaR4Test {
// Now the background scheduler will do its thing
myTerminologyDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
// The remaining versions and Code System entities should be gone now.
runInTransaction(() -> {

8
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java
View File

@ -45,6 +45,7 @@ import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
@ -109,6 +110,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
@ -128,6 +130,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.apache.commons.lang3.StringUtils.countMatches;
import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.hamcrest.MatcherAssert.assertThat;
@ -155,6 +159,9 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4Test.class);
@Autowired
private BatchJobHelper myBatchJobHelper;
@AfterEach
public final void after() {
myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences());
@ -356,6 +363,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test {
myResourceReindexingSvc.markAllResourcesForReindexing();
myResourceReindexingSvc.forceReindexingPass();
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runInTransaction(() -> {
assertEquals(3L, myTermConceptDao.count());

12
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoR5CodeSystemTest.java
View File

@ -2,22 +2,25 @@ package ca.uhn.fhir.jpa.dao.r5;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl;
import org.apache.commons.io.IOUtils;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r5.model.CodeSystem;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
public class FhirResourceDaoR5CodeSystemTest extends BaseJpaR5Test {
@Autowired private BatchJobHelper myBatchJobHelper;
@Test
public void testDeleteLargeCompleteCodeSystem() {
@ -42,6 +45,7 @@ public class FhirResourceDaoR5CodeSystemTest extends BaseJpaR5Test {
// Now the background scheduler will do its thing
myTermDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
runInTransaction(() -> {
assertEquals(0, myTermCodeSystemDao.count());
assertEquals(0, myTermCodeSystemVersionDao.count());
@ -100,6 +104,7 @@ public class FhirResourceDaoR5CodeSystemTest extends BaseJpaR5Test {
// Now the background scheduler will do its thing
myTermDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
// Entities for first resource should be gone now.
runInTransaction(() -> {
@ -134,6 +139,7 @@ public class FhirResourceDaoR5CodeSystemTest extends BaseJpaR5Test {
// Now the background scheduler will do its thing
myTermDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
// The remaining versions and Code System entities should be gone now.
runInTransaction(() -> {

7
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3CodeSystemTest.java
View File

@ -5,6 +5,7 @@ import ca.uhn.fhir.jpa.dao.dstu3.FhirResourceDaoDstu3TerminologyTest;
import ca.uhn.fhir.jpa.term.TermReindexingSvcImpl;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hl7.fhir.dstu3.model.BooleanType;
import org.hl7.fhir.dstu3.model.CodeSystem;
import org.hl7.fhir.dstu3.model.CodeType;
@ -19,16 +20,20 @@ import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.util.stream.Collectors;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
public class ResourceProviderDstu3CodeSystemTest extends BaseResourceProviderDstu3Test {
@Autowired private BatchJobHelper myBatchJobHelper;
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderDstu3CodeSystemTest.class);
public static FhirContext ourCtx = FhirContext.forDstu3Cached();
@ -133,6 +138,8 @@ public class ResourceProviderDstu3CodeSystemTest extends BaseResourceProviderDst
runInTransaction(() -> assertEquals(26L, myConceptDao.count()));
myTerminologyDeferredStorageSvc.saveDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
runInTransaction(() -> assertEquals(24L, myConceptDao.count()));
}

8
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcTest.java
View File

@ -5,12 +5,15 @@ import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.CodeType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.ArrayList;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
@ -18,6 +21,10 @@ public class TermCodeSystemStorageSvcTest extends BaseJpaR4Test {
public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
@Autowired
private BatchJobHelper myBatchJobHelper;
@Test
public void testStoreNewCodeSystemVersionForExistingCodeSystemNoVersionId() {
CodeSystem firstUpload = createCodeSystemWithMoreThan100Concepts();
@ -126,6 +133,7 @@ public class TermCodeSystemStorageSvcTest extends BaseJpaR4Test {
myTerminologyDeferredStorageSvc.setProcessDeferred(true);
myTerminologyDeferredStorageSvc.saveDeferred();
myTerminologyDeferredStorageSvc.setProcessDeferred(false);
myBatchJobHelper.awaitAllBulkJobCompletions(false, TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
assertEquals(theExpectedConceptCount, runInTransaction(() -> myTermConceptDao.count()));
}

21
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImplTest.java
View File

@ -10,10 +10,17 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.batch.core.JobExecution;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.same;
import static org.mockito.Mockito.times;
@ -33,6 +40,9 @@ public class TermDeferredStorageSvcImplTest {
@Mock
private ITermCodeSystemVersionDao myTermCodeSystemVersionDao;
@Mock
private JobExecution myJobExecution;
@Test
public void testSaveDeferredWithExecutionSuspended() {
TermDeferredStorageSvcImpl svc = new TermDeferredStorageSvcImpl();
@ -41,6 +51,17 @@ public class TermDeferredStorageSvcImplTest {
}
@Test
public void testStorageNotEmptyWhileJobsExecuting() {
TermDeferredStorageSvcImpl svc = new TermDeferredStorageSvcImpl();
ReflectionTestUtils.setField(svc, "myCurrentJobExecutions", Collections.singletonList(myJobExecution));
when(myJobExecution.isRunning()).thenReturn(true, false);
assertFalse(svc.isStorageQueueEmpty());
assertTrue(svc.isStorageQueueEmpty());
}
@Test
public void testSaveDeferred_Concept() {
TermConcept concept = new TermConcept();

10
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincJpaTest.java
View File

@ -5,20 +5,22 @@ import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hl7.fhir.r4.model.CodeSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.IOException;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_PRIMARY_DEFAULT;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.fail;
public class TerminologyLoaderSvcLoincJpaTest extends BaseJpaR4Test {
@Autowired private BatchJobHelper myBatchJobHelper;
private TermLoaderSvcImpl mySvc;
private ZipCollectionBuilder myFiles;
@BeforeEach
@ -62,6 +64,7 @@ public class TerminologyLoaderSvcLoincJpaTest extends BaseJpaR4Test {
mySvc.loadLoinc(myFiles.getFiles(), mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(false, TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME );
runInTransaction(() -> {
assertEquals(1, myTermCodeSystemDao.count());
@ -87,6 +90,7 @@ public class TerminologyLoaderSvcLoincJpaTest extends BaseJpaR4Test {
TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesWithPropertiesFileToZip(myFiles, "v268_loincupload.properties");
mySvc.loadLoinc(myFiles.getFiles(), mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(false, TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME );
runInTransaction(() -> {
assertEquals(1, myTermCodeSystemDao.count());

36
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplCurrentVersionR4Test.java
View File

@ -5,6 +5,7 @@ import ca.uhn.fhir.context.support.ValidationSupportContext;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.config.BaseConfig;
import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermValueSet;
@ -14,6 +15,7 @@ import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
@ -25,6 +27,7 @@ import org.hl7.fhir.r4.model.IdType;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.UriType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Answers;
@ -33,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.util.ResourceUtils;
import javax.persistence.EntityManager;
@ -50,6 +54,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT;
@ -75,6 +80,7 @@ import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UPLOAD_PROPERTIES_FILE;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_XML_FILE;
import static java.util.stream.Collectors.joining;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_ALL_VALUESET_ID;
@ -130,9 +136,11 @@ public class TerminologySvcImplCurrentVersionR4Test extends BaseJpaR4Test {
@Autowired
private ITermReadSvc myITermReadSvc;
@Autowired
@Qualifier(BaseConfig.JPA_VALIDATION_SUPPORT)
private IValidationSupport myJpaPersistedResourceValidationSupport;
@Autowired @Qualifier(BaseConfig.JPA_VALIDATION_SUPPORT)
private IValidationSupport myJpaPersistedResourceValidationSupport;
@Autowired private BatchJobHelper myBatchJobHelper;
private ZipCollectionBuilder myFiles;
private ServletRequestDetails myRequestDetails = new ServletRequestDetails();
@ -690,6 +698,7 @@ public class TerminologySvcImplCurrentVersionR4Test extends BaseJpaR4Test {
String currentVer = "2.68";
uploadLoincCodeSystem(currentVer, true);
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runCommonValidations(Lists.newArrayList(nonCurrentVer, currentVer));
@ -711,6 +720,7 @@ public class TerminologySvcImplCurrentVersionR4Test extends BaseJpaR4Test {
String lastCurrentVer = "2.69";
uploadLoincCodeSystem(lastCurrentVer, true);
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runCommonValidations(Lists.newArrayList(firstCurrentVer, noCurrentVer, lastCurrentVer));
@ -775,11 +785,27 @@ public class TerminologySvcImplCurrentVersionR4Test extends BaseJpaR4Test {
}
private TermCodeSystemVersion fetchCurrentCodeSystemVersion() {
runInTransaction(() -> {
List<TermCodeSystem> tcsList = myEntityManager.createQuery("from TermCodeSystem").getResultList();
List<TermCodeSystemVersion> tcsvList = myEntityManager.createQuery("from TermCodeSystemVersion").getResultList();
ourLog.error("tcslist: {}", tcsList.stream().map(tcs -> tcs.toString()).collect(joining("\n", "\n", "")));
ourLog.error("tcsvlist: {}", tcsvList.stream().map(v -> v.toString()).collect(joining("\n", "\n", "")));
if (tcsList.size() != 1) {
throw new IllegalStateException("More than one TCS: " +
tcsList.stream().map(tcs -> String.valueOf(tcs.getPid())).collect(joining()));
}
if (tcsList.get(0).getCurrentVersion() == null) {
throw new IllegalStateException("Current version is null in TCS: " + tcsList.get(0).getPid());
}
});
return runInTransaction(() -> (TermCodeSystemVersion) myEntityManager.createQuery(
"select tcsv from TermCodeSystemVersion tcsv join fetch tcsv.myCodeSystem tcs " +
"where tcs.myCurrentVersion = tcsv").getSingleResult());
"select tcsv from TermCodeSystemVersion tcsv join fetch tcsv.myCodeSystem tcs " +
"where tcs.myCurrentVersion = tcsv").getSingleResult());
}
private static void addBaseLoincMandatoryFilesToZip(
ZipCollectionBuilder theFiles, Boolean theIncludeTop2000, String theClassPathPrefix) throws IOException {
theFiles.addFileZip(theClassPathPrefix, LOINC_XML_FILE.getCode());

7
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java
View File

@ -8,6 +8,7 @@ import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermValueSet;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.CodeableConcept;
@ -18,6 +19,7 @@ import org.hl7.fhir.r4.model.codesystems.HttpVerb;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate;
@ -29,6 +31,7 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -39,6 +42,9 @@ import static org.junit.jupiter.api.Assertions.fail;
public class TerminologySvcImplR4Test extends BaseTermR4Test {
private static final Logger ourLog = LoggerFactory.getLogger(TerminologySvcImplR4Test.class);
@Autowired private BatchJobHelper myBatchJobHelper;
ConceptValidationOptions optsNoGuess = new ConceptValidationOptions();
ConceptValidationOptions optsGuess = new ConceptValidationOptions().setInferSystem(true);
@ -424,6 +430,7 @@ public class TerminologySvcImplR4Test extends BaseTermR4Test {
IIdType id_v2 = myCodeSystemDao.update(codeSystem, mySrd).getId().toUnqualified();
myTerminologyDeferredStorageSvc.saveAllDeferred();
myBatchJobHelper.awaitAllBulkJobCompletions(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
runInTransaction(() -> {
List<TermCodeSystemVersion> termCodeSystemVersions_updated = myTermCodeSystemVersionDao.findAll();

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ZipCollectionBuilder.java
View File

@ -28,7 +28,7 @@ public class ZipCollectionBuilder {
/**
* Constructor
*/
ZipCollectionBuilder() {
public ZipCollectionBuilder() {
myFiles = new ArrayList<>();
}

123
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/job/DynamicJobFlowSandbox.java Normal file
View File

@ -0,0 +1,123 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.job.builder.FlowBuilder;
import org.springframework.batch.core.job.builder.SimpleJobBuilder;
import org.springframework.batch.core.job.flow.Flow;
import org.springframework.batch.core.job.flow.support.SimpleFlow;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import java.util.List;
import java.util.stream.Collectors;
/**
* Not intended to ever run. Used as a sandbox for "interesting" jobs
*/
public class DynamicJobFlowSandbox {
protected static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemDeleteJobTest.class);
@Autowired
private JobBuilderFactory myJobBuilderFactory;
@Autowired
private StepBuilderFactory myStepBuilderFactory;
private List<Long> versionPidList = Lists.newArrayList(3L, 5L);
@Bean
public Job testJob() {
SimpleJobBuilder jobBuilder = myJobBuilderFactory.get("job")
.start(stepPreFlow());
// add a flow for each Pid
List<Flow> flowForEachPidList = versionPidList.stream().map(this::getFlowForPid).collect(Collectors.toList());
flowForEachPidList.forEach( flowForPid -> jobBuilder.on("COMPLETED").to(flowForPid) );
return jobBuilder.next(stepPostFlow()).build();
}
private Flow getFlowForPid(Long theLong) {
return new FlowBuilder<SimpleFlow>("flow-for-Pid-" + theLong)
.start(flowStep1(theLong))
.next(fllowStep2(theLong))
.build();
}
public Step flowStep1(long theLong) {
String name = "flow-step-1-for-Pid-" + theLong;
return myStepBuilderFactory.get(name)
.tasklet((contribution, chunkContext) -> {
ourLog.info("\n\n" + name + " executed\n\n");
return RepeatStatus.FINISHED;
})
.build();
}
public Step fllowStep2(long theLong) {
String name = "flow-step-2-for-Pid-" + theLong;
return myStepBuilderFactory.get(name)
.tasklet((contribution, chunkContext) -> {
ourLog.info("\n\n" + name + " executed\n\n");
return RepeatStatus.FINISHED;
})
.build();
}
public Step stepPreFlow() {
return myStepBuilderFactory.get("step-pre-flow")
.tasklet((contribution, chunkContext) -> {
ourLog.info("\n\nstep-pre-flow executed\n\n");
return RepeatStatus.FINISHED;
})
.build();
}
public Step stepPostFlow() {
return myStepBuilderFactory.get("step-post-flow")
.tasklet((contribution, chunkContext) -> {
ourLog.info("\n\nstep-post-flow executed\n\n");
return RepeatStatus.FINISHED;
})
.build();
}
}

266
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemDeleteJobTest.java Normal file
View File

@ -0,0 +1,266 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
import ca.uhn.fhir.jpa.term.UploadStatistics;
import ca.uhn.fhir.jpa.term.ZipCollectionBuilder;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.util.ResourceUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.Properties;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_ID;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_MAKE_CURRENT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_VERSION;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_TERMS_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_HIERARCHY_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PARENT_GROUP_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_PRIMARY_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_RSNA_PLAYBOOK_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UPLOAD_PROPERTIES_FILE;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_XML_FILE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TermCodeSystemDeleteJobTest extends BaseJpaR4Test {
private final ServletRequestDetails myRequestDetails = new ServletRequestDetails();
private Properties uploadProperties;
@Autowired private TermLoaderSvcImpl myTermLoaderSvc;
@Autowired private IBatchJobSubmitter myJobSubmitter;
@Autowired private BatchJobHelper myBatchJobHelper;
@Autowired @Qualifier(TERM_CODE_SYSTEM_DELETE_JOB_NAME)
private Job myTermCodeSystemDeleteJob;
private void initMultipleVersionLoad() throws Exception {
File file = ResourceUtils.getFile("classpath:loinc-ver/" + LOINC_UPLOAD_PROPERTIES_FILE.getCode());
uploadProperties = new Properties();
uploadProperties.load(new FileInputStream(file));
IFhirResourceDao<ValueSet> valueSetIFhirResourceDao = myDaoRegistry.getResourceDao(ValueSet.class);
}
@Test
public void runDeleteJobMultipleVersions() throws Exception {
initMultipleVersionLoad();
// loading a loinc CS with version loads two versions (second one with null version)
String firstCurrentVer = "2.67";
uploadLoincCodeSystem(firstCurrentVer, true);
long[] termCodeSystemPidVect = new long[1]; //bypass final restriction
runInTransaction(() -> {
assertEquals(1, myTermCodeSystemDao.count());
TermCodeSystem termCodeSystem = myTermCodeSystemDao.findByCodeSystemUri("http://loinc.org");
assertNotNull(termCodeSystem);
termCodeSystemPidVect[0] = termCodeSystem.getPid();
assertEquals(2, myTermCodeSystemVersionDao.count());
assertEquals(162, myTermConceptDao.count());
});
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_ID, new JobParameter(termCodeSystemPidVect[0], true) ));
JobExecution jobExecution = myJobSubmitter.runJob(myTermCodeSystemDeleteJob, jobParameters);
myBatchJobHelper.awaitJobCompletion(jobExecution);
assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode());
runInTransaction(() -> {
assertEquals(0, myTermCodeSystemDao.count());
assertNull(myTermCodeSystemDao.findByCodeSystemUri("http://loinc.org"));
assertEquals(0, myTermCodeSystemVersionDao.count());
assertEquals(0, myTermConceptDao.count());
});
}
@Test
public void runWithNoParameterFailsValidation() {
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemDeleteJob, new JobParameters())
);
assertEquals("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_ID + "'", thrown.getMessage());
}
@Test
public void runWithNullParameterFailsValidation() {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_ID, new JobParameter((Long) null, true) ));
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemDeleteJob, jobParameters)
);
assertEquals("'" + JOB_PARAM_CODE_SYSTEM_ID + "' parameter is null", thrown.getMessage());
}
@Test
public void runWithParameterZeroFailsValidation() {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_ID, new JobParameter(0L, true) ));
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemDeleteJob, jobParameters)
);
assertEquals("Invalid parameter '" + JOB_PARAM_CODE_SYSTEM_ID + "' value: 0", thrown.getMessage());
}
private IIdType uploadLoincCodeSystem(String theVersion, boolean theMakeItCurrent) throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
myRequestDetails.getUserData().put(LOINC_CODESYSTEM_MAKE_CURRENT, theMakeItCurrent);
uploadProperties.put(LOINC_CODESYSTEM_MAKE_CURRENT.getCode(), Boolean.toString(theMakeItCurrent));
assertTrue(
theVersion == null || theVersion.equals("2.67") || theVersion.equals("2.68") || theVersion.equals("2.69"),
"Version supported are: 2.67, 2.68, 2.69 and null" );
if (StringUtils.isBlank(theVersion)) {
uploadProperties.remove(LOINC_CODESYSTEM_VERSION.getCode());
} else {
uploadProperties.put(LOINC_CODESYSTEM_VERSION.getCode(), theVersion);
}
addLoincMandatoryFilesToZip(files, theVersion);
UploadStatistics stats = myTermLoaderSvc.loadLoinc(files.getFiles(), mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
return stats.getTarget();
}
public void addLoincMandatoryFilesToZip(ZipCollectionBuilder theFiles, String theVersion) throws IOException {
String theClassPathPrefix = getClassPathPrefix(theVersion);
addBaseLoincMandatoryFilesToZip(theFiles, true, theClassPathPrefix);
theFiles.addPropertiesZip(uploadProperties, LOINC_UPLOAD_PROPERTIES_FILE.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode());
}
private static void addBaseLoincMandatoryFilesToZip(
ZipCollectionBuilder theFiles, Boolean theIncludeTop2000, String theClassPathPrefix) throws IOException {
theFiles.addFileZip(theClassPathPrefix, LOINC_XML_FILE.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_GROUP_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_GROUP_TERMS_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PARENT_GROUP_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_HIERARCHY_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode());
if (theIncludeTop2000) {
theFiles.addFileZip(theClassPathPrefix, LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT.getCode());
}
}
private String getClassPathPrefix(String theVersion) {
String theClassPathPrefix = "/loinc-ver/v-no-version/";
if (StringUtils.isBlank(theVersion)) return theClassPathPrefix;
switch(theVersion) {
case "2.67":
return "/loinc-ver/v267/";
case "2.68":
return "/loinc-ver/v268/";
case "2.69":
return "/loinc-ver/v269/";
}
fail("Setup failed. Unexpected version: " + theVersion);
return null;
}
}

272
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/job/TermCodeSystemVersionDeleteJobTest.java Normal file
View File

@ -0,0 +1,272 @@
package ca.uhn.fhir.jpa.term.job;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2021 Smile CDR, Inc.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
import ca.uhn.fhir.jpa.term.UploadStatistics;
import ca.uhn.fhir.jpa.term.ZipCollectionBuilder;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.util.ResourceUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.Properties;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.JOB_PARAM_CODE_SYSTEM_VERSION_ID;
import static ca.uhn.fhir.jpa.batch.config.BatchConstants.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_MAKE_CURRENT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_VERSION;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DUPLICATE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_TERMS_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_HIERARCHY_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PARENT_GROUP_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_PRIMARY_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_RSNA_PLAYBOOK_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UPLOAD_PROPERTIES_FILE;
import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_XML_FILE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TermCodeSystemVersionDeleteJobTest extends BaseJpaR4Test {
private final ServletRequestDetails myRequestDetails = new ServletRequestDetails();
private Properties uploadProperties;
@Autowired private TermLoaderSvcImpl myTermLoaderSvc;
@Autowired private IBatchJobSubmitter myJobSubmitter;
@Autowired private BatchJobHelper myBatchJobHelper;
@Autowired @Qualifier(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME)
private Job myTermCodeSystemVersionDeleteJob;
private void initMultipleVersionLoad() throws Exception {
File file = ResourceUtils.getFile("classpath:loinc-ver/" + LOINC_UPLOAD_PROPERTIES_FILE.getCode());
uploadProperties = new Properties();
uploadProperties.load(new FileInputStream(file));
IFhirResourceDao<ValueSet> valueSetIFhirResourceDao = myDaoRegistry.getResourceDao(ValueSet.class);
}
@Test
public void runDeleteJobDeleteOneVersion() throws Exception {
initMultipleVersionLoad();
String firstCurrentVer = "2.67";
uploadLoincCodeSystem(firstCurrentVer, true);
long[] termCodeSystemVersionPidVect = new long[1]; //bypass final restriction
runInTransaction(() -> {
assertEquals(1, myTermCodeSystemDao.count());
TermCodeSystem termCodeSystem = myTermCodeSystemDao.findByCodeSystemUri("http://loinc.org");
assertNotNull(termCodeSystem);
TermCodeSystemVersion termCodeSystemVersion = myTermCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(termCodeSystem.getPid());
assertNotNull(termCodeSystemVersion);
termCodeSystemVersionPidVect[0] = termCodeSystemVersion.getPid();
assertEquals(2, myTermCodeSystemVersionDao.count());
assertEquals(81 * 2, myTermConceptDao.count());
});
JobParameters jobParameters = new JobParameters(Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_VERSION_ID, new JobParameter(termCodeSystemVersionPidVect[0], true) ));
JobExecution jobExecution = myJobSubmitter.runJob(myTermCodeSystemVersionDeleteJob, jobParameters);
myBatchJobHelper.awaitJobCompletion(jobExecution);
assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode());
runInTransaction(() -> {
assertEquals(1, myTermCodeSystemDao.count());
assertNotNull(myTermCodeSystemDao.findByCodeSystemUri("http://loinc.org"));
assertEquals(1, myTermCodeSystemVersionDao.count());
assertEquals(81, myTermConceptDao.count());
});
}
@Test
public void runWithNoParameterFailsValidation() {
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemVersionDeleteJob, new JobParameters())
);
assertEquals("This job needs Parameter: '" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "'", thrown.getMessage());
}
@Test
public void runWithNullParameterFailsValidation() {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_VERSION_ID, new JobParameter((Long) null, true) ));
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemVersionDeleteJob, jobParameters)
);
assertEquals("'" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "' parameter is null", thrown.getMessage());
}
@Test
public void runWithParameterZeroFailsValidation() {
JobParameters jobParameters = new JobParameters(
Collections.singletonMap(
JOB_PARAM_CODE_SYSTEM_VERSION_ID, new JobParameter(0L, true) ));
JobParametersInvalidException thrown = Assertions.assertThrows(
JobParametersInvalidException.class,
() -> myJobSubmitter.runJob(myTermCodeSystemVersionDeleteJob, jobParameters)
);
assertEquals("Invalid parameter '" + JOB_PARAM_CODE_SYSTEM_VERSION_ID + "' value: 0", thrown.getMessage());
}
private IIdType uploadLoincCodeSystem(String theVersion, boolean theMakeItCurrent) throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
myRequestDetails.getUserData().put(LOINC_CODESYSTEM_MAKE_CURRENT, theMakeItCurrent);
uploadProperties.put(LOINC_CODESYSTEM_MAKE_CURRENT.getCode(), Boolean.toString(theMakeItCurrent));
assertTrue(
theVersion == null || theVersion.equals("2.67") || theVersion.equals("2.68") || theVersion.equals("2.69"),
"Version supported are: 2.67, 2.68, 2.69 and null" );
if (StringUtils.isBlank(theVersion)) {
uploadProperties.remove(LOINC_CODESYSTEM_VERSION.getCode());
} else {
uploadProperties.put(LOINC_CODESYSTEM_VERSION.getCode(), theVersion);
}
addLoincMandatoryFilesToZip(files, theVersion);
UploadStatistics stats = myTermLoaderSvc.loadLoinc(files.getFiles(), mySrd);
myTerminologyDeferredStorageSvc.saveAllDeferred();
return stats.getTarget();
}
public void addLoincMandatoryFilesToZip(ZipCollectionBuilder theFiles, String theVersion) throws IOException {
String theClassPathPrefix = getClassPathPrefix(theVersion);
addBaseLoincMandatoryFilesToZip(theFiles, true, theClassPathPrefix);
theFiles.addPropertiesZip(uploadProperties, LOINC_UPLOAD_PROPERTIES_FILE.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode());
}
private static void addBaseLoincMandatoryFilesToZip(
ZipCollectionBuilder theFiles, Boolean theIncludeTop2000, String theClassPathPrefix) throws IOException {
theFiles.addFileZip(theClassPathPrefix, LOINC_XML_FILE.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_GROUP_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_GROUP_TERMS_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PARENT_GROUP_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_HIERARCHY_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode());
if (theIncludeTop2000) {
theFiles.addFileZip(theClassPathPrefix, LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode());
theFiles.addFileZip(theClassPathPrefix, LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT.getCode());
}
}
private String getClassPathPrefix(String theVersion) {
String theClassPathPrefix = "/loinc-ver/v-no-version/";
if (StringUtils.isBlank(theVersion)) return theClassPathPrefix;
switch(theVersion) {
case "2.67":
return "/loinc-ver/v267/";
case "2.68":
return "/loinc-ver/v268/";
case "2.69":
return "/loinc-ver/v269/";
}
fail("Setup failed. Unexpected version: " + theVersion);
return null;
}
}

27
hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/BatchJobHelper.java
View File

@ -48,17 +48,30 @@ public class BatchJobHelper {
myJobExplorer = theJobExplorer;
}
public List<JobExecution> awaitAllBulkJobCompletions(String... theJobNames) {
return awaitAllBulkJobCompletions(true, theJobNames);
}
/**
* Await and report for job completions
* @param theFailIfNotJobsFound indicate if must fail in case no matching jobs are found
* @param theJobNames The job names to match
* @return the matched JobExecution(s)
*/
public List<JobExecution> awaitAllBulkJobCompletions(boolean theFailIfNotJobsFound, String... theJobNames) {
assert theJobNames.length > 0;
List<JobInstance> matchingJobInstances = new ArrayList<>();
for (String nextName : theJobNames) {
matchingJobInstances.addAll(myJobExplorer.findJobInstancesByJobName(nextName, 0, 100));
}
if (matchingJobInstances.isEmpty()) {
List<String> wantNames = Arrays.asList(theJobNames);
List<String> haveNames = myJobExplorer.getJobNames();
fail("There are no jobs running - Want names " + wantNames + " and have names " + haveNames);
if (theFailIfNotJobsFound) {
if (matchingJobInstances.isEmpty()) {
List<String> wantNames = Arrays.asList(theJobNames);
List<String> haveNames = myJobExplorer.getJobNames();
fail("There are no jobs running - Want names " + wantNames + " and have names " + haveNames);
}
}
List<JobExecution> matchingExecutions = matchingJobInstances.stream().flatMap(jobInstance -> myJobExplorer.getJobExecutions(jobInstance).stream()).collect(Collectors.toList());
awaitJobCompletions(matchingExecutions);
@ -82,7 +95,11 @@ public class BatchJobHelper {
await().atMost(120, TimeUnit.SECONDS).until(() -> {
JobExecution jobExecution = myJobExplorer.getJobExecution(theJobExecution.getId());
ourLog.info("JobExecution {} currently has status: {}- Failures if any: {}", theJobExecution.getId(), jobExecution.getStatus(), jobExecution.getFailureExceptions());
return jobExecution.getStatus() == BatchStatus.COMPLETED || jobExecution.getStatus() == BatchStatus.FAILED;
// JM: Adding ABANDONED status because given the description, it s similar to FAILURE, and we need to avoid tests failing because
// of wait timeouts caused by unmatched statuses. Also adding STOPPED because tests were found where this wait timed out
// with jobs keeping that status during the whole wait
return jobExecution.getStatus() == BatchStatus.COMPLETED || jobExecution.getStatus() == BatchStatus.FAILED
|| jobExecution.getStatus() == BatchStatus.ABANDONED || jobExecution.getStatus() == BatchStatus.STOPPED;
});
}