From a8489f4331fb7467ff96192ce7b0e3c289b19df3 Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Mon, 29 Jul 2019 19:30:46 -0400 Subject: [PATCH 1/4] Added TRM_VALUESET.EXPANSION_STATUS column, index, fields to entity, and migration tasks. --- .../ca/uhn/fhir/jpa/entity/TermValueSet.java | 19 ++++++++++++ .../TermValueSetExpansionStatusEnum.java | 29 +++++++++++++++++++ .../jpa/term/TerminologySvcImplR4Test.java | 2 +- .../tasks/HapiFhirJpaMigrationTasks.java | 5 ++++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java index 723472895bc..28a60a1fef8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java @@ -43,6 +43,7 @@ import static org.apache.commons.lang3.StringUtils.length; public class TermValueSet implements Serializable { private static final long serialVersionUID = 1L; + public static final int MAX_EXPANSION_STATUS_LENGTH = 50; public static final int MAX_NAME_LENGTH = 200; public static final int MAX_URL_LENGTH = 200; @@ -68,6 +69,15 @@ public class TermValueSet implements Serializable { @OneToMany(mappedBy = "myValueSet") private List myConcepts; + @Enumerated(EnumType.STRING) + @Column(name = "EXPANSION_STATUS", nullable = false, length = MAX_EXPANSION_STATUS_LENGTH) + private TermValueSetExpansionStatusEnum myExpansionStatus; + + public TermValueSet() { + super(); + myExpansionStatus = TermValueSetExpansionStatusEnum.NOT_EXPANDED; + } + public Long getId() { return myId; } @@ -110,6 +120,14 @@ public class TermValueSet implements Serializable { return myConcepts; } + public TermValueSetExpansionStatusEnum getExpansionStatus() { + return myExpansionStatus; + } + + public void setExpansionStatus(TermValueSetExpansionStatusEnum theExpansionStatus) { + myExpansionStatus = theExpansionStatus; + } + @Override public boolean equals(Object theO) { if (this == theO) return true; @@ -139,6 +157,7 @@ public class TermValueSet implements Serializable { .append("myResourcePid", myResourcePid) .append("myName", myName) .append(myConcepts != null ? ("myConcepts - size=" + myConcepts.size()) : ("myConcepts=(null)")) + .append("myExpansionStatus", myExpansionStatus) .toString(); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java new file mode 100644 index 00000000000..05a64e63341 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java @@ -0,0 +1,29 @@ +package ca.uhn.fhir.jpa.entity; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public enum TermValueSetExpansionStatusEnum { + + NOT_EXPANDED, + EXPANSION_IN_PROGRESS, + EXPANDED + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java index dbd4a91eeea..f7eaa42c369 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java @@ -7,7 +7,6 @@ import ca.uhn.fhir.jpa.entity.*; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.server.IBundleProvider; -import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.param.UriParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; @@ -964,6 +963,7 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", valueSet.getUrl()); assertEquals("Terminology Services Connectation #1 Extensional case #2", valueSet.getName()); assertEquals(codeSystem.getConcept().size(), valueSet.getConcepts().size()); + assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, valueSet.getExpansionStatus()); TermValueSetConcept concept = valueSet.getConcepts().get(0); ourLog.info("Code:\n" + concept.toString()); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index 36659301668..49e6cfb439c 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -115,6 +115,11 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { .toColumn("RES_ID") .references("HFJ_RESOURCE", "RES_ID"); termValueSetTable.addColumn("NAME").nullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, TermValueSet.MAX_NAME_LENGTH); + termValueSetTable.addColumn("EXPANSION_STATUS").nonNullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, TermValueSet.MAX_EXPANSION_STATUS_LENGTH); + termValueSetTable + .addIndex("IDX_VALUESET_EXP_STATUS") + .unique(false) + .withColumns("EXPANSION_STATUS"); // TermValueSetConcept version.startSectionWithMessage("Processing table: TRM_VALUESET_CONCEPT"); From 0e2dcce841076b6f97a4aa32c608b80dc2761916 Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Wed, 31 Jul 2019 15:52:25 -0400 Subject: [PATCH 2/4] Incremental work on large ValueSet expansion support; still need to fix asynchronous tests and actually use the terminology tables when expanding. --- .../jpa/dao/data/ITermValueSetConceptDao.java | 7 +- .../ITermValueSetConceptDesignationDao.java | 2 +- .../fhir/jpa/dao/data/ITermValueSetDao.java | 8 +- .../dstu3/FhirResourceDaoValueSetDstu3.java | 2 +- .../jpa/dao/r4/FhirResourceDaoValueSetR4.java | 2 +- .../jpa/term/BaseHapiTerminologySvcImpl.java | 135 +++++---- .../jpa/term/HapiTerminologySvcDstu2.java | 8 +- .../jpa/term/HapiTerminologySvcDstu3.java | 17 +- .../fhir/jpa/term/HapiTerminologySvcR4.java | 9 +- .../fhir/jpa/term/IHapiTerminologySvc.java | 8 +- ....java => IValueSetConceptAccumulator.java} | 8 +- .../jpa/term/ValueSetConceptAccumulator.java | 141 ++++++++++ ...nsionComponentWithConceptAccumulator.java} | 8 +- .../jpa/term/TerminologySvcImplR4Test.java | 262 ++++++++++++++---- ...xtensional-case-3-cs-with-designations.xml | 9 + .../extensional-case-3-vs-with-exclude.xml | 31 +++ 16 files changed, 509 insertions(+), 148 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/{IValueSetCodeAccumulator.java => IValueSetConceptAccumulator.java} (71%) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/{ValueSetExpansionComponentWithCodeAccumulator.java => ValueSetExpansionComponentWithConceptAccumulator.java} (80%) create mode 100644 hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-vs-with-exclude.xml diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDao.java index e613c17022b..9577c47b842 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDao.java @@ -26,10 +26,15 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import java.util.Optional; + public interface ITermValueSetConceptDao extends JpaRepository { @Query("DELETE FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid") @Modifying - void deleteTermValueSetConceptsByValueSetId(@Param("pid") Long theValueSetId); + void deleteByTermValueSetId(@Param("pid") Long theValueSetId); + + @Query("SELECT vsc FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid AND vsc.mySystem = :system_url AND vsc.myCode = :codeval") + Optional findByValueSetIdSystemAndCode(@Param("pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDesignationDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDesignationDao.java index 6861c86f59a..1a0875b5a2f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDesignationDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptDesignationDao.java @@ -30,6 +30,6 @@ public interface ITermValueSetConceptDesignationDao extends JpaRepository { @Query("DELETE FROM TermValueSet vs WHERE vs.myId = :pid") @Modifying - void deleteTermValueSetById(@Param("pid") Long theId); + void deleteByTermValueSetId(@Param("pid") Long theId); @Query("SELECT vs FROM TermValueSet vs WHERE vs.myResourcePid = :resource_pid") Optional findByResourcePid(@Param("resource_pid") Long theResourcePid); @@ -40,4 +43,7 @@ public interface ITermValueSetDao extends JpaRepository { @Query("SELECT vs FROM TermValueSet vs WHERE vs.myUrl = :url") Optional findByUrl(@Param("url") String theUrl); + @Query("SELECT vs FROM TermValueSet vs WHERE vs.myExpansionStatus = :expansion_status") + Page findByExpansionStatus(Pageable pageable, @Param("expansion_status") TermValueSetExpansionStatusEnum theExpansionStatus); + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java index d5cc4ef233c..282b1ab5a19 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java @@ -315,7 +315,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 try { ValueSet valueSet = (ValueSet) theResource; org.hl7.fhir.r4.model.ValueSet converted = VersionConvertor_30_40.convertValueSet(valueSet); - myHapiTerminologySvc.storeTermValueSetAndChildren(retVal, converted); + myHapiTerminologySvc.storeTermValueSet(retVal, converted); } catch (FHIRException fe) { throw new InternalErrorException(fe); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java index 4de557a0b34..0663d8cd3f7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java @@ -313,7 +313,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4 imple if (myDaoConfig.isPreExpandValueSetsExperimental()) { if (retVal.getDeleted() == null) { ValueSet valueSet = (ValueSet) theResource; - myHapiTerminologySvc.storeTermValueSetAndChildren(retVal, valueSet); + myHapiTerminologySvc.storeTermValueSet(retVal, valueSet); } else { myHapiTerminologySvc.deleteValueSetAndChildren(retVal); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index a3cbf53253d..bc2bb5874cb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -101,6 +101,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, private static boolean ourLastResultsFromTranslationCache; // For testing. private static boolean ourLastResultsFromTranslationWithReverseCache; // For testing. @Autowired + protected DaoRegistry myDaoRegistry; + @Autowired protected ITermCodeSystemDao myCodeSystemDao; @Autowired protected ITermConceptDao myConceptDao; @@ -151,8 +153,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, private PlatformTransactionManager myTransactionManager; @Autowired(required = false) private IFulltextSearchSvc myFulltextSearchSvc; + @Autowired + private PlatformTransactionManager myTxManager; - private void addCodeIfNotAlreadyAdded(IValueSetCodeAccumulator theValueSetCodeAccumulator, Set theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter) { + private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter) { String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); String code = theConcept.getCode(); String display = theConcept.getDisplay(); @@ -160,28 +164,28 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, theCodeCounter, codeSystem, code, display); } - private void addCodeIfNotAlreadyAdded(IValueSetCodeAccumulator theValueSetCodeAccumulator, Set theAddedCodes, Collection theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCode, String theDisplay) { + private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, Collection theDesignations, boolean theAdd, AtomicInteger theCodeCounter, String theCodeSystem, String theCode, String theDisplay) { if (isNoneBlank(theCodeSystem, theCode)) { if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) { - theValueSetCodeAccumulator.includeCodeWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations); + theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations); theCodeCounter.incrementAndGet(); } if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) { - theValueSetCodeAccumulator.excludeCode(theCodeSystem, theCode); + theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); theCodeCounter.decrementAndGet(); } } } - private void addConceptsToList(IValueSetCodeAccumulator theValueSetCodeAccumulator, Set theAddedCodes, String theSystem, List theConcept, boolean theAdd) { + private void addConceptsToList(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, String theSystem, List theConcept, boolean theAdd) { for (CodeSystem.ConceptDefinitionComponent next : theConcept) { if (isNoneBlank(theSystem, next.getCode())) { if (theAdd && theAddedCodes.add(theSystem + "|" + next.getCode())) { - theValueSetCodeAccumulator.includeCode(theSystem, next.getCode(), next.getDisplay()); + theValueSetCodeAccumulator.includeConcept(theSystem, next.getCode(), next.getDisplay()); } if (!theAdd && theAddedCodes.remove(theSystem + "|" + next.getCode())) { - theValueSetCodeAccumulator.excludeCode(theSystem, next.getCode()); + theValueSetCodeAccumulator.excludeConcept(theSystem, next.getCode()); } } addConceptsToList(theValueSetCodeAccumulator, theAddedCodes, theSystem, next.getConcept(), theAdd); @@ -389,9 +393,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get(); ourLog.info("Deleting existing TermValueSet {} and its children...", existingTermValueSet.getId()); - myValueSetConceptDesignationDao.deleteTermValueSetConceptDesignationsByValueSetId(existingTermValueSet.getId()); - myValueSetConceptDao.deleteTermValueSetConceptsByValueSetId(existingTermValueSet.getId()); - myValueSetDao.deleteTermValueSetById(existingTermValueSet.getId()); + myValueSetConceptDesignationDao.deleteByTermValueSetId(existingTermValueSet.getId()); + myValueSetConceptDao.deleteByTermValueSetId(existingTermValueSet.getId()); + myValueSetDao.deleteByTermValueSetId(existingTermValueSet.getId()); ourLog.info("Done deleting existing TermValueSet {} and its children.", existingTermValueSet.getId()); ourLog.info("Flushing..."); @@ -457,7 +461,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, @Transactional(propagation = Propagation.REQUIRED) public ValueSet expandValueSet(ValueSet theValueSetToExpand) { - ValueSetExpansionComponentWithCodeAccumulator expansionComponent = new ValueSetExpansionComponentWithCodeAccumulator(); + ValueSetExpansionComponentWithConceptAccumulator expansionComponent = new ValueSetExpansionComponentWithConceptAccumulator(); expansionComponent.setIdentifier(UUID.randomUUID().toString()); expansionComponent.setTimestamp(new Date()); @@ -476,11 +480,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, @Override @Transactional(propagation = Propagation.REQUIRED) - public void expandValueSet(ValueSet theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator) { + public void expandValueSet(ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) { expandValueSet(theValueSetToExpand, theValueSetCodeAccumulator, new AtomicInteger(0)); } - private void expandValueSet(ValueSet theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator, AtomicInteger theCodeCounter) { + private void expandValueSet(ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, AtomicInteger theCodeCounter) { Set addedCodes = new HashSet<>(); // Handle includes @@ -509,7 +513,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, return retVal; } - private void expandValueSetHandleIncludeOrExclude(IValueSetCodeAccumulator theValueSetCodeAccumulator, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter) { + private void expandValueSetHandleIncludeOrExclude(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, boolean theAdd, AtomicInteger theCodeCounter) { String system = theInclude.getSystem(); boolean hasSystem = isNotBlank(system); boolean hasValueSet = theInclude.getValueSet().size() > 0; @@ -676,10 +680,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode); if (code != null) { if (theAdd && theAddedCodes.add(system + "|" + nextCode)) { - theValueSetCodeAccumulator.includeCode(system, nextCode, code.getDisplay()); + theValueSetCodeAccumulator.includeConcept(system, nextCode, code.getDisplay()); } if (!theAdd && theAddedCodes.remove(system + "|" + nextCode)) { - theValueSetCodeAccumulator.excludeCode(system, nextCode); + theValueSetCodeAccumulator.excludeConcept(system, nextCode); } } } @@ -706,7 +710,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } if (isNoneBlank(nextConcept.getSystem(), nextConcept.getCode()) && !theAdd && theAddedCodes.remove(nextConcept.getSystem() + "|" + nextConcept.getCode())) { - theValueSetCodeAccumulator.excludeCode(nextConcept.getSystem(), nextConcept.getCode()); + theValueSetCodeAccumulator.excludeConcept(nextConcept.getSystem(), nextConcept.getCode()); } } @@ -716,10 +720,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } } - private void expandWithoutHibernateSearch(IValueSetCodeAccumulator theValueSetCodeAccumulator, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) { + private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) { ourLog.trace("Hibernate search is not enabled"); - if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithCodeAccumulator) { - Validate.isTrue(((ValueSetExpansionComponentWithCodeAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server."); + if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) { + Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server."); } Validate.isTrue(theInclude.getFilter().isEmpty(), "Can not expand ValueSet with filters - Hibernate Search is not enabled on this server."); Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server."); @@ -1491,9 +1495,46 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, ourLog.info("Done storing TermConceptMap."); } + @Scheduled(fixedDelay = 1000) + @Transactional + @Override + public synchronized void preExpandValueSetToTerminologyTables() { + new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus theStatus) { + Optional optionalTermValueSet = getNextTermValueSetNotExpanded(); + if (optionalTermValueSet.isPresent()) { + TermValueSet termValueSet = optionalTermValueSet.get(); + termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANSION_IN_PROGRESS); + myValueSetDao.saveAndFlush(termValueSet); + + ValueSet valueSet = getValueSetFromResourceTable(termValueSet.getResource()); + + expandValueSet(valueSet, new ValueSetConceptAccumulator(termValueSet, myValueSetConceptDao, myValueSetConceptDesignationDao)); + + termValueSet.setExpansionStatus(TermValueSetExpansionStatusEnum.EXPANDED); + myValueSetDao.saveAndFlush(termValueSet); + } + } + }); + } + + protected abstract ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable); + + private Optional getNextTermValueSetNotExpanded() { + Optional retVal = Optional.empty(); + Page page = myValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetExpansionStatusEnum.NOT_EXPANDED); + + if (!page.getContent().isEmpty()) { + retVal = Optional.of(page.getContent().get(0)); + } + + return retVal; + } + @Override @Transactional - public void storeTermValueSetAndChildren(ResourceTable theResourceTable, ValueSet theValueSet) { + public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) { ourLog.info("Storing TermValueSet {}", theValueSet.getIdElement().getValue()); ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied"); @@ -1513,58 +1554,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, String url = termValueSet.getUrl(); Optional optionalExistingTermValueSetByUrl = myValueSetDao.findByUrl(url); if (!optionalExistingTermValueSetByUrl.isPresent()) { + myValueSetDao.save(termValueSet); - int conceptsSaved = 0; - int designationsSaved = 0; - - // FIXME: DM 2019-07-15 - Here we should call expandValueSet(ValueSet theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator). - // FIXME: DM 2019-07-15 - We need an implementation IValueSetCodeAccumulator that saves ValueSetConcept records and their children. - ValueSet expandedValueSet = expandValueSet(theValueSet); - if (expandedValueSet.hasExpansion()) { - if (expandedValueSet.getExpansion().hasTotal() && expandedValueSet.getExpansion().getTotal() > 0) { - TermValueSetConcept concept; - for (ValueSet.ValueSetExpansionContainsComponent contains : expandedValueSet.getExpansion().getContains()) { - ValidateUtil.isNotBlankOrThrowInvalidRequest(contains.getSystem(), "ValueSet contains a concept with no system value"); - ValidateUtil.isNotBlankOrThrowInvalidRequest(contains.getCode(), "ValueSet contains a concept with no code value"); - - concept = new TermValueSetConcept(); - concept.setValueSet(termValueSet); - concept.setSystem(contains.getSystem()); - concept.setCode(contains.getCode()); - concept.setDisplay(contains.hasDisplay() ? contains.getDisplay() : null); - myValueSetConceptDao.save(concept); - - TermValueSetConceptDesignation designation; - for (ValueSet.ConceptReferenceDesignationComponent containedDesignation : contains.getDesignation()) { - ValidateUtil.isNotBlankOrThrowInvalidRequest(containedDesignation.getValue(), "ValueSet contains a concept designation with no value"); - - designation = new TermValueSetConceptDesignation(); - designation.setConcept(concept); - designation.setLanguage(containedDesignation.hasLanguage() ? containedDesignation.getLanguage() : null); - if (containedDesignation.hasUse()) { - designation.setUseSystem(containedDesignation.getUse().hasSystem() ? containedDesignation.getUse().getSystem() : null); - designation.setUseCode(containedDesignation.getUse().hasCode() ? containedDesignation.getUse().getCode() : null); - designation.setUseDisplay(containedDesignation.getUse().hasDisplay() ? containedDesignation.getUse().getDisplay() : null); - } - designation.setValue(containedDesignation.getValue()); - myValueSetConceptDesignationDao.save(designation); - - if (designationsSaved++ % 250 == 0) { - ourLog.info("Have pre-expanded {} designations in ValueSet", designationsSaved); - myValueSetConceptDesignationDao.flush(); - } - } - - // TODO: DM 2019-07-16 - We need TermValueSetConceptProperty, similar to TermConceptProperty. - // TODO: DM 2019-07-16 - We should also populate TermValueSetConceptProperty entities here. - - if (conceptsSaved++ % 250 == 0) { - ourLog.info("Have pre-expanded {} concepts in ValueSet", conceptsSaved); - myValueSetConceptDao.flush(); - } - } - } - } } else { TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java index 5d750cb00a9..498f25dfe94 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu2.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.term; * #L% */ +import ca.uhn.fhir.jpa.model.entity.ResourceTable; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.hapi.validation.IValidationSupport; @@ -81,13 +82,18 @@ public class HapiTerminologySvcDstu2 extends BaseHapiTerminologySvcImpl { return null; } + @Override + protected ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) { + throw new UnsupportedOperationException(); + } + @Override public IBaseResource expandValueSet(IBaseResource theValueSetToExpand) { throw new UnsupportedOperationException(); } @Override - public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator) { + public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) { throw new UnsupportedOperationException(); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java index 48f5cd600a0..da83782a810 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java @@ -4,6 +4,7 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem; import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.util.CoverageIgnore; import ca.uhn.fhir.util.UrlUtil; @@ -177,7 +178,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen } @Override - public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator) { + public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) { ValueSet valueSetToExpand = (ValueSet) theValueSetToExpand; try { @@ -292,6 +293,20 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen } } + @Override + protected org.hl7.fhir.r4.model.ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) { + ValueSet valueSet = myValueSetResourceDao.toResource(ValueSet.class, theResourceTable, null, false); + + org.hl7.fhir.r4.model.ValueSet valueSetR4; + try { + valueSetR4 = VersionConvertor_30_40.convertValueSet(valueSet); + } catch (FHIRException e) { + throw new InternalErrorException(e); + } + + return valueSetR4; + } + @Override public boolean isCodeSystemSupported(FhirContext theContext, String theSystem) { return myTerminologySvc.supportsSystem(theSystem); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java index 1e1bbe84b96..799ceecd46c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java @@ -3,9 +3,9 @@ package ca.uhn.fhir.jpa.term; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.util.CoverageIgnore; import ca.uhn.fhir.util.UrlUtil; -import ca.uhn.fhir.util.ValidateUtil; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.hapi.ctx.IValidationSupport; @@ -138,7 +138,7 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements } @Override - public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator) { + public void expandValueSet(IBaseResource theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) { ValueSet valueSetToExpand = (ValueSet) theValueSetToExpand; super.expandValueSet(valueSetToExpand, theValueSetCodeAccumulator); } @@ -231,6 +231,11 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements return myValidationSupport.fetchCodeSystem(myContext, theSystem); } + @Override + protected ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) { + return myValueSetResourceDao.toResource(ValueSet.class, theResourceTable, null, false); + } + @Override public boolean isCodeSystemSupported(FhirContext theContext, String theSystem) { return supportsSystem(theSystem); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java index 61d7151e1b9..5faf0f72320 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologySvc.java @@ -44,14 +44,14 @@ public interface IHapiTerminologySvc { ValueSet expandValueSet(ValueSet theValueSetToExpand); - void expandValueSet(ValueSet theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator); + void expandValueSet(ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator); /** * Version independent */ IBaseResource expandValueSet(IBaseResource theValueSetToExpand); - void expandValueSet(IBaseResource theValueSetToExpand, IValueSetCodeAccumulator theValueSetCodeAccumulator); + void expandValueSet(IBaseResource theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator); List expandValueSet(String theValueSet); @@ -94,7 +94,7 @@ public interface IHapiTerminologySvc { void storeTermConceptMapAndChildren(ResourceTable theResourceTable, ConceptMap theConceptMap); - void storeTermValueSetAndChildren(ResourceTable theResourceTable, ValueSet theValueSet); + void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet); boolean supportsSystem(String theCodeSystem); @@ -107,4 +107,6 @@ public interface IHapiTerminologySvc { AtomicInteger applyDeltaCodesystemsAdd(String theSystem, @Nullable String theParent, CodeSystem theValue); AtomicInteger applyDeltaCodesystemsRemove(String theSystem, CodeSystem theDelta); + + void preExpandValueSetToTerminologyTables(); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetCodeAccumulator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java similarity index 71% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetCodeAccumulator.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java index a97608cb094..47dc7000c23 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetCodeAccumulator.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IValueSetConceptAccumulator.java @@ -24,12 +24,12 @@ import ca.uhn.fhir.jpa.entity.TermConceptDesignation; import java.util.Collection; -public interface IValueSetCodeAccumulator { +public interface IValueSetConceptAccumulator { - void includeCode(String theSystem, String theCode, String theDisplay); + void includeConcept(String theSystem, String theCode, String theDisplay); - void includeCodeWithDesignations(String theSystem, String theCode, String theDisplay, Collection theDesignations); + void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection theDesignations); - void excludeCode(String theSystem, String theCode); + void excludeConcept(String theSystem, String theCode); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java new file mode 100644 index 00000000000..4e2e5b92ec2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetConceptAccumulator.java @@ -0,0 +1,141 @@ +package ca.uhn.fhir.jpa.term; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao; +import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao; +import ca.uhn.fhir.jpa.entity.TermConceptDesignation; +import ca.uhn.fhir.jpa.entity.TermValueSet; +import ca.uhn.fhir.jpa.entity.TermValueSetConcept; +import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation; +import ca.uhn.fhir.util.ValidateUtil; + +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.Optional; + +import static org.apache.commons.lang3.StringUtils.*; + +public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator { + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ValueSetConceptAccumulator.class); + + private TermValueSet myTermValueSet; + private ITermValueSetConceptDao myValueSetConceptDao; + private ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao; + private int myConceptsSaved; + private int myDesignationsSaved; + + public ValueSetConceptAccumulator(@Nonnull TermValueSet theTermValueSet, @Nonnull ITermValueSetConceptDao theValueSetConceptDao, @Nonnull ITermValueSetConceptDesignationDao theValueSetConceptDesignationDao) { + myTermValueSet = theTermValueSet; + myValueSetConceptDao = theValueSetConceptDao; + myValueSetConceptDesignationDao = theValueSetConceptDesignationDao; + myConceptsSaved = 0; + myDesignationsSaved = 0; + } + + @Override + public void includeConcept(String theSystem, String theCode, String theDisplay) { + saveConcept(theSystem, theCode, theDisplay); + } + + @Override + public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection theDesignations) { + TermValueSetConcept concept = saveConcept(theSystem, theCode, theDisplay); + for (TermConceptDesignation designation : theDesignations) { + saveConceptDesignation(concept, designation); + } + } + + @Override + public void excludeConcept(String theSystem, String theCode) { + if (isAnyBlank(theSystem, theCode)) { + return; + } + + // Get existing entity so it can be deleted. + Optional optionalConcept = myValueSetConceptDao.findByValueSetIdSystemAndCode(myTermValueSet.getId(), theSystem, theCode); + + if (optionalConcept.isPresent()) { + TermValueSetConcept concept = optionalConcept.get(); + + ourLog.info("Excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl()); + for (TermValueSetConceptDesignation designation : concept.getDesignations()) { + myValueSetConceptDesignationDao.deleteById(designation.getId()); + } + myValueSetConceptDao.deleteById(concept.getId()); + ourLog.info("Done excluding [{}|{}] from ValueSet[{}]", concept.getSystem(), concept.getCode(), myTermValueSet.getUrl()); + + ourLog.info("Flushing..."); + myValueSetConceptDesignationDao.flush(); + myValueSetConceptDao.flush(); + ourLog.info("Done flushing."); + } + } + + private TermValueSetConcept saveConcept(String theSystem, String theCode, String theDisplay) { + ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "ValueSet contains a concept with no system value"); + ValidateUtil.isNotBlankOrThrowInvalidRequest(theCode, "ValueSet contains a concept with no code value"); + + TermValueSetConcept concept = new TermValueSetConcept(); + concept.setValueSet(myTermValueSet); + concept.setSystem(theSystem); + concept.setCode(theCode); + if (isNotBlank(theDisplay)) { + concept.setDisplay(theDisplay); + } + myValueSetConceptDao.save(concept); + + if (myConceptsSaved++ % 250 == 0) { + ourLog.info("Have pre-expanded {} concepts in ValueSet[{}]", myConceptsSaved, myTermValueSet.getUrl()); + myValueSetConceptDao.flush(); + } + + return concept; + } + + private TermValueSetConceptDesignation saveConceptDesignation(TermValueSetConcept theConcept, TermConceptDesignation theDesignation) { + ValidateUtil.isNotBlankOrThrowInvalidRequest(theDesignation.getValue(), "ValueSet contains a concept designation with no value"); + + TermValueSetConceptDesignation designation = new TermValueSetConceptDesignation(); + designation.setConcept(theConcept); + designation.setLanguage(theDesignation.getLanguage()); + if (isNoneBlank(theDesignation.getUseSystem(), theDesignation.getUseCode())) { + designation.setUseSystem(theDesignation.getUseSystem()); + designation.setUseCode(theDesignation.getUseCode()); + if (isNotBlank(theDesignation.getUseDisplay())) { + designation.setUseDisplay(theDesignation.getUseDisplay()); + } + } + designation.setValue(theDesignation.getValue()); + myValueSetConceptDesignationDao.save(designation); + + if (myDesignationsSaved++ % 250 == 0) { + ourLog.info("Have pre-expanded {} designations in ValueSet[{}]", myDesignationsSaved, myTermValueSet.getUrl()); + myValueSetConceptDesignationDao.flush(); + } + + return designation; + } + + // TODO: DM 2019-07-16 - We need TermValueSetConceptProperty, similar to TermConceptProperty. + // TODO: DM 2019-07-16 - We should also populate TermValueSetConceptProperty entities here. + // TODO: DM 2019-07-30 - Expansions don't include the properties themselves; they are needed to facilitate filters and parameterized expansions. +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithCodeAccumulator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java similarity index 80% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithCodeAccumulator.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java index eb94f91d403..5aa95013365 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithCodeAccumulator.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/ValueSetExpansionComponentWithConceptAccumulator.java @@ -27,10 +27,10 @@ import org.hl7.fhir.r4.model.ValueSet; import java.util.Collection; @Block() -public class ValueSetExpansionComponentWithCodeAccumulator extends ValueSet.ValueSetExpansionComponent implements IValueSetCodeAccumulator { +public class ValueSetExpansionComponentWithConceptAccumulator extends ValueSet.ValueSetExpansionComponent implements IValueSetConceptAccumulator { @Override - public void includeCode(String theSystem, String theCode, String theDisplay) { + public void includeConcept(String theSystem, String theCode, String theDisplay) { ValueSet.ValueSetExpansionContainsComponent contains = this.addContains(); contains.setSystem(theSystem); contains.setCode(theCode); @@ -38,7 +38,7 @@ public class ValueSetExpansionComponentWithCodeAccumulator extends ValueSet.Valu } @Override - public void includeCodeWithDesignations(String theSystem, String theCode, String theDisplay, Collection theDesignations) { + public void includeConceptWithDesignations(String theSystem, String theCode, String theDisplay, Collection theDesignations) { ValueSet.ValueSetExpansionContainsComponent contains = this.addContains(); contains.setSystem(theSystem); contains.setCode(theCode); @@ -58,7 +58,7 @@ public class ValueSetExpansionComponentWithCodeAccumulator extends ValueSet.Valu } @Override - public void excludeCode(String theSystem, String theCode) { + public void excludeConcept(String theSystem, String theCode) { this .getContains() .removeIf(t -> diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java index 82e6886ca75..fde3c5d854c 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java @@ -11,6 +11,7 @@ import ca.uhn.fhir.rest.param.UriParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; import ca.uhn.fhir.util.TestUtil; +import com.google.common.collect.Lists; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.hapi.ctx.IValidationSupport; import org.hl7.fhir.r4.model.*; @@ -42,7 +43,7 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { @Rule public final ExpectedException expectedException = ExpectedException.none(); @Mock - IValueSetCodeAccumulator myValueSetCodeAccumulator; + IValueSetConceptAccumulator myValueSetCodeAccumulator; private IIdType myConceptMapId; private IIdType myExtensionalCsId; private IIdType myExtensionalVsId; @@ -135,6 +136,11 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { loadAndPersistValueSet(); } + private void loadAndPersistCodeSystemAndValueSetWithDesignationsAndExclude() throws IOException { + loadAndPersistCodeSystemWithDesignations(); + loadAndPersistValueSetWithExclude(); + } + private void loadAndPersistCodeSystem() throws IOException { CodeSystem codeSystem = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml"); persistCodeSystem(codeSystem); @@ -159,6 +165,11 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { persistValueSet(valueSet); } + private void loadAndPersistValueSetWithExclude() throws IOException { + ValueSet valueSet = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs-with-exclude.xml"); + persistValueSet(valueSet); + } + private void persistValueSet(ValueSet theValueSet) { new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() { @Override @@ -589,7 +600,7 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { include.setSystem(CS_URL); myTermSvc.expandValueSet(vs, myValueSetCodeAccumulator); - verify(myValueSetCodeAccumulator, times(9)).includeCodeWithDesignations(anyString(), anyString(), nullable(String.class), anyCollection()); + verify(myValueSetCodeAccumulator, times(9)).includeConceptWithDesignations(anyString(), anyString(), nullable(String.class), anyCollection()); } @Test @@ -625,15 +636,24 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { TermConcept concept = concepts.get(0); assertEquals("8450-9", concept.getCode()); assertEquals("Systolic blood pressure--expiration", concept.getDisplay()); - assertEquals(1, concept.getDesignations().size()); + assertEquals(2, concept.getDesignations().size()); - TermConceptDesignation designation = concept.getDesignations().iterator().next(); + List designations = Lists.newArrayList(concept.getDesignations().iterator()); + + TermConceptDesignation designation = designations.get(0); assertEquals("nl", designation.getLanguage()); assertEquals("http://snomed.info/sct", designation.getUseSystem()); assertEquals("900000000000013009", designation.getUseCode()); assertEquals("Synonym", designation.getUseDisplay()); assertEquals("Systolische bloeddruk - expiratie", designation.getValue()); + designation = designations.get(1); + assertEquals("sv", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systoliskt blodtryck - utgång", designation.getValue()); + concept = concepts.get(1); assertEquals("11378-7", concept.getCode()); assertEquals("Systolic blood pressure at First encounter", concept.getDisplay()); @@ -948,67 +968,197 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { ValueSet valueSet = myValueSetDao.read(myExtensionalVsId); ourLog.info("ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(valueSet)); - new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theStatus) { - Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); - assertTrue(optionalValueSetByResourcePid.isPresent()); + runInTransaction(()->{ + Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); + assertTrue(optionalValueSetByResourcePid.isPresent()); - Optional optionalValueSetByUrl = myTermValueSetDao.findByUrl("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"); - assertTrue(optionalValueSetByUrl.isPresent()); + Optional optionalValueSetByUrl = myTermValueSetDao.findByUrl("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"); + assertTrue(optionalValueSetByUrl.isPresent()); - TermValueSet valueSet = optionalValueSetByUrl.get(); - assertSame(optionalValueSetByResourcePid.get(), valueSet); - ourLog.info("ValueSet:\n" + valueSet.toString()); - assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", valueSet.getUrl()); - assertEquals("Terminology Services Connectation #1 Extensional case #2", valueSet.getName()); - assertEquals(codeSystem.getConcept().size(), valueSet.getConcepts().size()); - assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, valueSet.getExpansionStatus()); + TermValueSet termValueSet = optionalValueSetByUrl.get(); + assertSame(optionalValueSetByResourcePid.get(), termValueSet); + ourLog.info("ValueSet:\n" + termValueSet.toString()); + assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); + assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); + assertEquals(0, termValueSet.getConcepts().size()); // DM 2019-07-30: Concepts are persisted in the background. + assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, termValueSet.getExpansionStatus()); + }); - TermValueSetConcept concept = valueSet.getConcepts().get(0); - ourLog.info("Code:\n" + concept.toString()); - assertEquals("http://acme.org", concept.getSystem()); - assertEquals("8450-9", concept.getCode()); - assertEquals("Systolic blood pressure--expiration", concept.getDisplay()); - assertEquals(1, concept.getDesignations().size()); + // FIXME: DM 2019-07-31 - Wait for ValueSet expansion status to be EXPANDED. + try { + Thread.sleep(10000); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } - TermValueSetConceptDesignation designation = concept.getDesignations().get(0); - assertEquals("nl", designation.getLanguage()); - assertEquals("http://snomed.info/sct", designation.getUseSystem()); - assertEquals("900000000000013009", designation.getUseCode()); - assertEquals("Synonym", designation.getUseDisplay()); - assertEquals("Systolische bloeddruk - expiratie", designation.getValue()); + runInTransaction(()->{ + Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); + assertTrue(optionalValueSetByResourcePid.isPresent()); - concept = valueSet.getConcepts().get(1); - ourLog.info("Code:\n" + concept.toString()); - assertEquals("http://acme.org", concept.getSystem()); - assertEquals("11378-7", concept.getCode()); - assertEquals("Systolic blood pressure at First encounter", concept.getDisplay()); - assertEquals(0, concept.getDesignations().size()); + Optional optionalValueSetByUrl = myTermValueSetDao.findByUrl("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"); + assertTrue(optionalValueSetByUrl.isPresent()); - // ... + TermValueSet termValueSet = optionalValueSetByUrl.get(); + assertSame(optionalValueSetByResourcePid.get(), termValueSet); + ourLog.info("ValueSet:\n" + termValueSet.toString()); + assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); + assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); + assertEquals(codeSystem.getConcept().size(), termValueSet.getConcepts().size()); + assertEquals(TermValueSetExpansionStatusEnum.EXPANDED, termValueSet.getExpansionStatus()); - concept = valueSet.getConcepts().get(22); - ourLog.info("Code:\n" + concept.toString()); - assertEquals("http://acme.org", concept.getSystem()); - assertEquals("8491-3", concept.getCode()); - assertEquals("Systolic blood pressure 1 hour minimum", concept.getDisplay()); - assertEquals(1, concept.getDesignations().size()); + TermValueSetConcept concept = termValueSet.getConcepts().get(0); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8450-9", concept.getCode()); + assertEquals("Systolic blood pressure--expiration", concept.getDisplay()); + assertEquals(2, concept.getDesignations().size()); - designation = concept.getDesignations().get(0); - assertEquals("nl", designation.getLanguage()); - assertEquals("http://snomed.info/sct", designation.getUseSystem()); - assertEquals("900000000000013009", designation.getUseCode()); - assertEquals("Synonym", designation.getUseDisplay()); - assertEquals("Systolische bloeddruk minimaal 1 uur", designation.getValue()); + TermValueSetConceptDesignation designation = concept.getDesignations().get(0); + assertEquals("nl", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systolische bloeddruk - expiratie", designation.getValue()); - concept = valueSet.getConcepts().get(23); - ourLog.info("Code:\n" + concept.toString()); - assertEquals("http://acme.org", concept.getSystem()); - assertEquals("8492-1", concept.getCode()); - assertEquals("Systolic blood pressure 8 hour minimum", concept.getDisplay()); - assertEquals(0, concept.getDesignations().size()); - } + designation = concept.getDesignations().get(1); + assertEquals("sv", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systoliskt blodtryck - utgång", designation.getValue()); + + concept = termValueSet.getConcepts().get(1); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("11378-7", concept.getCode()); + assertEquals("Systolic blood pressure at First encounter", concept.getDisplay()); + assertEquals(0, concept.getDesignations().size()); + + // ... + + concept = termValueSet.getConcepts().get(22); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8491-3", concept.getCode()); + assertEquals("Systolic blood pressure 1 hour minimum", concept.getDisplay()); + assertEquals(1, concept.getDesignations().size()); + + designation = concept.getDesignations().get(0); + assertEquals("nl", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systolische bloeddruk minimaal 1 uur", designation.getValue()); + + concept = termValueSet.getConcepts().get(23); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8492-1", concept.getCode()); + assertEquals("Systolic blood pressure 8 hour minimum", concept.getDisplay()); + assertEquals(0, concept.getDesignations().size()); + }); + } + + @Test + public void testStoreTermValueSetAndChildrenWithExclude() throws Exception { + myDaoConfig.setPreExpandValueSetsExperimental(true); + + loadAndPersistCodeSystemAndValueSetWithDesignationsAndExclude(); + + CodeSystem codeSystem = myCodeSystemDao.read(myExtensionalCsId); + ourLog.info("CodeSystem:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(codeSystem)); + + ValueSet valueSet = myValueSetDao.read(myExtensionalVsId); + ourLog.info("ValueSet:\n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(valueSet)); + + runInTransaction(()->{ + Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); + assertTrue(optionalValueSetByResourcePid.isPresent()); + + Optional optionalValueSetByUrl = myTermValueSetDao.findByUrl("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"); + assertTrue(optionalValueSetByUrl.isPresent()); + + TermValueSet termValueSet = optionalValueSetByUrl.get(); + assertSame(optionalValueSetByResourcePid.get(), termValueSet); + ourLog.info("ValueSet:\n" + termValueSet.toString()); + assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); + assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); + assertEquals(0, termValueSet.getConcepts().size()); // DM 2019-07-30: Concepts are persisted in the background. + assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, termValueSet.getExpansionStatus()); + }); + + // FIXME: DM 2019-07-31 - Wait for ValueSet expansion status to be EXPANDED. + try { + Thread.sleep(10000); + } catch (InterruptedException theE) { + theE.printStackTrace(); + } + + runInTransaction(()->{ + Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); + assertTrue(optionalValueSetByResourcePid.isPresent()); + + Optional optionalValueSetByUrl = myTermValueSetDao.findByUrl("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"); + assertTrue(optionalValueSetByUrl.isPresent()); + + TermValueSet termValueSet = optionalValueSetByUrl.get(); + assertSame(optionalValueSetByResourcePid.get(), termValueSet); + ourLog.info("ValueSet:\n" + termValueSet.toString()); + assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); + assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); + assertEquals(codeSystem.getConcept().size() - 2, termValueSet.getConcepts().size()); + assertEquals(TermValueSetExpansionStatusEnum.EXPANDED, termValueSet.getExpansionStatus()); + + TermValueSetConcept concept = termValueSet.getConcepts().get(0); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8450-9", concept.getCode()); + assertEquals("Systolic blood pressure--expiration", concept.getDisplay()); + assertEquals(2, concept.getDesignations().size()); + + TermValueSetConceptDesignation designation = concept.getDesignations().get(0); + assertEquals("nl", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systolische bloeddruk - expiratie", designation.getValue()); + + designation = concept.getDesignations().get(1); + assertEquals("sv", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systoliskt blodtryck - utgång", designation.getValue()); + + concept = termValueSet.getConcepts().get(1); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("11378-7", concept.getCode()); + assertEquals("Systolic blood pressure at First encounter", concept.getDisplay()); + assertEquals(0, concept.getDesignations().size()); + + // ... + + concept = termValueSet.getConcepts().get(22 - 2); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8491-3", concept.getCode()); + assertEquals("Systolic blood pressure 1 hour minimum", concept.getDisplay()); + assertEquals(1, concept.getDesignations().size()); + + designation = concept.getDesignations().get(0); + assertEquals("nl", designation.getLanguage()); + assertEquals("http://snomed.info/sct", designation.getUseSystem()); + assertEquals("900000000000013009", designation.getUseCode()); + assertEquals("Synonym", designation.getUseDisplay()); + assertEquals("Systolische bloeddruk minimaal 1 uur", designation.getValue()); + + concept = termValueSet.getConcepts().get(23 - 2); + ourLog.info("Code:\n" + concept.toString()); + assertEquals("http://acme.org", concept.getSystem()); + assertEquals("8492-1", concept.getCode()); + assertEquals("Systolic blood pressure 8 hour minimum", concept.getDisplay()); + assertEquals(0, concept.getDesignations().size()); }); } diff --git a/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-cs-with-designations.xml b/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-cs-with-designations.xml index e3bad86cb07..19cc0e071e6 100644 --- a/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-cs-with-designations.xml +++ b/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-cs-with-designations.xml @@ -12,6 +12,15 @@ + + + + + + + + + diff --git a/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-vs-with-exclude.xml b/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-vs-with-exclude.xml new file mode 100644 index 00000000000..70529a456f6 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/extensional-case-3-vs-with-exclude.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ffda92514a401f05d89c85cfdcdc0446c6e8be4c Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Wed, 31 Jul 2019 16:33:50 -0400 Subject: [PATCH 3/4] Incremental work on large ValueSet expansion support; still need to actually use the terminology tables when expanding. --- .../jpa/term/BaseHapiTerminologySvcImpl.java | 2 +- .../jpa/term/TerminologySvcImplR4Test.java | 22 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index bc2bb5874cb..5d0ccb49ccb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -1495,7 +1495,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, ourLog.info("Done storing TermConceptMap."); } - @Scheduled(fixedDelay = 1000) + @Scheduled(fixedDelay = 600000) // 10 minutes. @Transactional @Override public synchronized void preExpandValueSetToTerminologyTables() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java index fde3c5d854c..fef85b39cc8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplR4Test.java @@ -22,6 +22,7 @@ import org.junit.rules.ExpectedException; import org.mockito.Mock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.test.context.TestPropertySource; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; @@ -38,6 +39,9 @@ import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +@TestPropertySource(properties = { + "scheduling_disabled=true" +}) public class TerminologySvcImplR4Test extends BaseJpaR4Test { private static final Logger ourLog = LoggerFactory.getLogger(TerminologySvcImplR4Test.class); @Rule @@ -980,16 +984,11 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { ourLog.info("ValueSet:\n" + termValueSet.toString()); assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); - assertEquals(0, termValueSet.getConcepts().size()); // DM 2019-07-30: Concepts are persisted in the background. + assertEquals(0, termValueSet.getConcepts().size()); assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, termValueSet.getExpansionStatus()); }); - // FIXME: DM 2019-07-31 - Wait for ValueSet expansion status to be EXPANDED. - try { - Thread.sleep(10000); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } + myTermSvc.preExpandValueSetToTerminologyTables(); runInTransaction(()->{ Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); @@ -1083,16 +1082,11 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test { ourLog.info("ValueSet:\n" + termValueSet.toString()); assertEquals("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2", termValueSet.getUrl()); assertEquals("Terminology Services Connectation #1 Extensional case #2", termValueSet.getName()); - assertEquals(0, termValueSet.getConcepts().size()); // DM 2019-07-30: Concepts are persisted in the background. + assertEquals(0, termValueSet.getConcepts().size()); assertEquals(TermValueSetExpansionStatusEnum.NOT_EXPANDED, termValueSet.getExpansionStatus()); }); - // FIXME: DM 2019-07-31 - Wait for ValueSet expansion status to be EXPANDED. - try { - Thread.sleep(10000); - } catch (InterruptedException theE) { - theE.printStackTrace(); - } + myTermSvc.preExpandValueSetToTerminologyTables(); runInTransaction(()->{ Optional optionalValueSetByResourcePid = myTermValueSetDao.findByResourcePid(myExtensionalVsId.getIdPartAsLong()); From 78842164fd29e7156520b220b2adec6c8d657b03 Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk Date: Thu, 1 Aug 2019 13:37:46 -0400 Subject: [PATCH 4/4] Addressing review comments. --- .../ca/uhn/fhir/jpa/entity/TermValueSetConcept.java | 8 ++++++-- .../jpa/entity/TermValueSetExpansionStatusEnum.java | 13 +++++++++++++ .../fhir/jpa/term/BaseHapiTerminologySvcImpl.java | 1 - .../migrate/tasks/HapiFhirJpaMigrationTasks.java | 11 +++++++---- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java index 2f2293b1b41..4827e756d89 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java @@ -35,8 +35,12 @@ import java.util.List; import static org.apache.commons.lang3.StringUtils.left; import static org.apache.commons.lang3.StringUtils.length; -@Table(name = "TRM_VALUESET_CONCEPT", indexes = { - @Index(name = "IDX_VALUESET_CONCEPT_CS_CD", columnList = "SYSTEM_URL, CODEVAL") +/* + * DM 2019-08-01 - Do not use IDX_VALUESET_CONCEPT_CS_CD; this was previously used as an index so reusing the name will + * bork up migration tasks. + */ +@Table(name = "TRM_VALUESET_CONCEPT", uniqueConstraints = { + @UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CD", columnNames = {"VALUESET_PID", "SYSTEM_URL", "CODEVAL"}) }) @Entity() public class TermValueSetConcept implements Serializable { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java index 05a64e63341..33cb4301097 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetExpansionStatusEnum.java @@ -20,10 +20,23 @@ package ca.uhn.fhir.jpa.entity; * #L% */ +/** + * This enum is used to indicate the expansion status of a given ValueSet in the terminology tables. In this context, + * an expanded ValueSet has its included concepts stored in the terminology tables as well. + */ public enum TermValueSetExpansionStatusEnum { + /** + * This status indicates the ValueSet is waiting to be picked up and expanded by a scheduled task. + */ NOT_EXPANDED, + /** + * This status indicates the ValueSet has been picked up by a scheduled task and is mid-expansion. + */ EXPANSION_IN_PROGRESS, + /** + * This status indicates the ValueSet has been picked up by a scheduled task and expansion is complete. + */ EXPANDED } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index 5d0ccb49ccb..c574f84a122 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -1496,7 +1496,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } @Scheduled(fixedDelay = 600000) // 10 minutes. - @Transactional @Override public synchronized void preExpandValueSetToTerminologyTables() { new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index 49e6cfb439c..fbc526bda52 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -133,15 +133,18 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { .references("TRM_VALUESET", "PID"); termValueSetConceptTable.addColumn("SYSTEM_URL").nonNullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, TermCodeSystem.MAX_URL_LENGTH); termValueSetConceptTable.addColumn("CODEVAL").nonNullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, TermConcept.MAX_CODE_LENGTH); - termValueSetConceptTable - .addIndex("IDX_VALUESET_CONCEPT_CS_CD") - .unique(false) - .withColumns("SYSTEM_URL", "CODEVAL"); termValueSetConceptTable.addColumn("DISPLAY").nullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, TermConcept.MAX_DESC_LENGTH); version.onTable("TRM_VALUESET_CONCEPT") .renameColumn("CODE", "CODEVAL", true, true) .renameColumn("SYSTEM", "SYSTEM_URL", true, true); + version.startSectionWithMessage("Processing table: TRM_VALUESET_CONCEPT, swapping index for unique constraint"); + termValueSetConceptTable.dropIndex("IDX_VALUESET_CONCEPT_CS_CD"); + termValueSetConceptTable + .addIndex("IDX_VS_CONCEPT_CS_CD") + .unique(true) + .withColumns("VALUESET_PID", "SYSTEM_URL", "CODEVAL"); + // TermValueSetConceptDesignation version.startSectionWithMessage("Processing table: TRM_VALUESET_C_DESIGNATION"); version.addIdGenerator("SEQ_VALUESET_C_DSGNTN_PID");