diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java index 6985f9e71a4..86514b5ae53 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java @@ -39,9 +39,9 @@ import java.util.Map.Entry; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index ceffc80e905..4b643375b78 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -92,9 +92,9 @@ import static org.apache.commons.lang3.StringUtils.*; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java index ce72e7228c3..6a734512d70 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java @@ -10,6 +10,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import java.util.Date; import java.util.List; /* @@ -21,9 +22,9 @@ import java.util.List; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,21 +35,42 @@ import java.util.List; public interface ITermConceptDao extends JpaRepository { + @Query("SELECT COUNT(t) FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid") + Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid); + + /** + * Used in Smile CDR - Do not delete + */ + @SuppressWarnings("unused") + @Query("SELECT COUNT(*) FROM TermConcept t WHERE t.myUpdated > :cutoff") + long countUpdatedAfter(@Param("cutoff") Date theFullTextIndexedUntil); + + /** + * Used in Smile CDR - Do not delete + */ + @SuppressWarnings("unused") + @Query("SELECT t FROM TermConcept t ORDER BY t.myUpdated ASC") + Slice findAllOrderedByLastUpdated(Pageable thePage); + @Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system AND c.myCode = :code") TermConcept findByCodeSystemAndCode(@Param("code_system") TermCodeSystemVersion theCodeSystem, @Param("code") String theCode); - @Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system") - List findByCodeSystemVersion(@Param("code_system") TermCodeSystemVersion theCodeSystem); - @Query("SELECT t FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid") Slice findByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid); - @Query("SELECT COUNT(t) FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid") - Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid); + @Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system") + List findByCodeSystemVersion(@Param("code_system") TermCodeSystemVersion theCodeSystem); @Query("SELECT t FROM TermConcept t WHERE t.myIndexStatus = null") Page findResourcesRequiringReindexing(Pageable thePageRequest); + /** + * Used in Smile CDR - Do not delete + */ + @SuppressWarnings("unused") + @Query("SELECT t FROM TermConcept t WHERE t.myUpdated > :cutoff ORDER BY t.myUpdated ASC") + Slice findUpdatedAfterOrderedByLastUpdated(Pageable thePage, @Param("cutoff") Date theFullTextIndexedUntil); + @Query("UPDATE TermConcept t SET t.myIndexStatus = null") @Modifying int markAllForReindexing(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java index d60a96ad8f0..d3cb23d9896 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java @@ -18,9 +18,9 @@ import java.util.Collection; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java index a715cc02679..b672aca316d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java @@ -45,7 +45,8 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; @Table(name = "TRM_CONCEPT", uniqueConstraints = { @UniqueConstraint(name = "IDX_CONCEPT_CS_CODE", columnNames = {"CODESYSTEM_PID", "CODE"}) }, indexes = { - @Index(name = "IDX_CONCEPT_INDEXSTATUS", columnList = "INDEX_STATUS") + @Index(name = "IDX_CONCEPT_INDEXSTATUS", columnList = "INDEX_STATUS"), + @Index(name = "IDX_CONCEPT_UPDATED", columnList = "CONCEPT_UPDATED") }) public class TermConcept implements Serializable { protected static final int MAX_DESC_LENGTH = 400; @@ -59,15 +60,15 @@ public class TermConcept implements Serializable { @Column(name = "CODE", length = 100, nullable = false) @Fields({@Field(name = "myCode", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "exactAnalyzer")),}) private String myCode; - + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "CONCEPT_UPDATED", nullable = true) + private Date myUpdated; @ManyToOne() @JoinColumn(name = "CODESYSTEM_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPT_PID_CS_PID")) private TermCodeSystemVersion myCodeSystem; - @Column(name = "CODESYSTEM_PID", insertable = false, updatable = false) @Fields({@Field(name = "myCodeSystemVersionPid")}) private long myCodeSystemVersionPid; - @Column(name = "DISPLAY", length = MAX_DESC_LENGTH, nullable = true) @Fields({ @Field(name = "myDisplay", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "standardAnalyzer")), @@ -76,15 +77,12 @@ public class TermConcept implements Serializable { @Field(name = "myDisplayPhonetic", index = org.hibernate.search.annotations.Index.YES, store = Store.NO, analyze = Analyze.YES, analyzer = @Analyzer(definition = "autocompletePhoneticAnalyzer")) }) private String myDisplay; - @OneToMany(mappedBy = "myConcept", orphanRemoval = false) @Field(name = "PROPmyProperties", analyzer = @Analyzer(definition = "termConceptPropertyAnalyzer")) @FieldBridge(impl = TermConceptPropertyFieldBridge.class) private Collection myProperties; - @OneToMany(mappedBy = "myConcept", orphanRemoval = false) private Collection myDesignations; - @Id() @SequenceGenerator(name = "SEQ_CONCEPT_PID", sequenceName = "SEQ_CONCEPT_PID") @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CONCEPT_PID") @@ -92,18 +90,17 @@ public class TermConcept implements Serializable { private Long myId; @Column(name = "INDEX_STATUS", nullable = true) private Long myIndexStatus; - @Transient @Field(name = "myParentPids", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "conceptParentPidsAnalyzer")) + @Lob + @Column(name="PARENT_PIDS", nullable = true) private String myParentPids; @OneToMany(cascade = {}, fetch = FetchType.LAZY, mappedBy = "myChild") private Collection myParents; @Column(name = "CODE_SEQUENCE", nullable = true) private Integer mySequence; - public TermConcept() { super(); } - public TermConcept(TermCodeSystemVersion theCs, String theCode) { setCodeSystemVersion(theCs); setCode(theCode); @@ -296,6 +293,14 @@ public class TermConcept implements Serializable { return null; } + public Date getUpdated() { + return myUpdated; + } + + public void setUpdated(Date theUpdated) { + myUpdated = theUpdated; + } + @Override public int hashCode() { HashCodeBuilder b = new HashCodeBuilder(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index a7554c0b114..7fec0b67e5b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -327,26 +327,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } - private void doDelete(String theDescriptor, Supplier> theLoader, Supplier theCounter, JpaRepository theDao) { - int count; - ourLog.info(" * Deleting {}", theDescriptor); - int totalCount = theCounter.get(); - StopWatch sw = new StopWatch(); - count = 0; - while (true) { - Slice link = theLoader.get(); - if (link.hasContent() == false) { - break; - } - - theDao.deleteInBatch(link); - - count += link.getNumberOfElements(); - ourLog.info(" * {} {} deleted - {}/sec - ETA: {}", count, theDescriptor, sw.formatThroughput(count, TimeUnit.SECONDS), sw.getEstimatedTimeRemaining(count, totalCount)); - } - theDao.flush(); - } - public void deleteConceptMap(ResourceTable theResourceTable) { // Get existing entity so it can be deleted. Optional optionalExistingTermConceptMapById = myConceptMapDao.findTermConceptMapByResourcePid(theResourceTable.getId()); @@ -388,6 +368,26 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, deleteConceptMap(theResourceTable); } + private void doDelete(String theDescriptor, Supplier> theLoader, Supplier theCounter, JpaRepository theDao) { + int count; + ourLog.info(" * Deleting {}", theDescriptor); + int totalCount = theCounter.get(); + StopWatch sw = new StopWatch(); + count = 0; + while (true) { + Slice link = theLoader.get(); + if (link.hasContent() == false) { + break; + } + + theDao.deleteInBatch(link); + + count += link.getNumberOfElements(); + ourLog.info(" * {} {} deleted - {}/sec - ETA: {}", count, theDescriptor, sw.formatThroughput(count, TimeUnit.SECONDS), sw.getEstimatedTimeRemaining(count, totalCount)); + } + theDao.flush(); + } + private int ensureParentsSaved(Collection theParents) { ourLog.trace("Checking {} parents", theParents.size()); int retVal = 0; @@ -397,6 +397,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, TermConcept nextParent = nextLink.getParent(); retVal += ensureParentsSaved(nextParent.getParents()); if (nextParent.getId() == null) { + nextParent.setUpdated(new Date()); myConceptDao.saveAndFlush(nextParent); retVal++; ourLog.debug("Saved parent code {} and got id {}", nextParent.getCode(), nextParent.getId()); @@ -887,9 +888,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, for (TermConcept nextConcept : concepts) { - StringBuilder parentsBuilder = new StringBuilder(); - createParentsString(parentsBuilder, nextConcept.getId()); - nextConcept.setParentPids(parentsBuilder.toString()); + if (isBlank(nextConcept.getParentPidsAsString())) { + StringBuilder parentsBuilder = new StringBuilder(); + createParentsString(parentsBuilder, nextConcept.getId()); + nextConcept.setParentPids(parentsBuilder.toString()); + } saveConcept(nextConcept); count++; @@ -923,6 +926,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, if (theConcept.getId() == null || theConcept.getIndexStatus() == null) { retVal++; theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED); + theConcept.setUpdated(new Date()); myConceptDao.save(theConcept); for (TermConceptProperty next : theConcept.getProperties()) { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java index 2fe5f229028..0375a20d664 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java @@ -104,6 +104,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest { @Qualifier("myConceptMapDaoR4") protected IFhirResourceDaoConceptMap myConceptMapDao; @Autowired + protected ITermConceptDao myTermConceptDao; + @Autowired @Qualifier("myConditionDaoR4") protected IFhirResourceDao myConditionDao; @Autowired diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java index f43abf9f44c..9a86b6e8a2d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java @@ -1,30 +1,13 @@ package ca.uhn.fhir.jpa.dao.r4; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsStringIgnoringCase; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -import java.util.*; - -import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl; -import org.hl7.fhir.r4.model.*; -import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceCategory; -import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceClinicalStatus; -import org.hl7.fhir.r4.model.CodeSystem.CodeSystemContentMode; -import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; -import org.hl7.fhir.r4.model.ValueSet.*; -import org.hl7.fhir.instance.model.api.IIdType; -import org.junit.*; -import org.springframework.beans.factory.annotation.Autowired; - import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; import ca.uhn.fhir.jpa.dao.SearchParameterMap; -import ca.uhn.fhir.jpa.entity.*; +import ca.uhn.fhir.jpa.entity.ResourceTable; +import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; +import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; +import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl; import ca.uhn.fhir.jpa.term.IHapiTerminologySvc; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.param.TokenParam; @@ -34,20 +17,36 @@ import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; import ca.uhn.fhir.util.TestUtil; import ca.uhn.fhir.validation.FhirValidator; import ca.uhn.fhir.validation.ValidationResult; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceCategory; +import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceClinicalStatus; +import org.hl7.fhir.r4.model.CodeSystem.CodeSystemContentMode; +import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; +import org.hl7.fhir.r4.model.ValueSet.*; +import org.junit.*; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4TerminologyTest.class); public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system"; public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set"; - + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4TerminologyTest.class); @Autowired private IHapiTerminologySvc myHapiTerminologySvc; @After public void after() { myDaoConfig.setDeferIndexingForCodesystemsOfSize(new DaoConfig().getDeferIndexingForCodesystemsOfSize()); - + BaseHapiTerminologySvcImpl.setForceSaveDeferredAlwaysForUnitTest(false); } @@ -96,38 +95,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { TermConcept childCA = new TermConcept(cs, "childCA").setDisplay("Child CA"); parentC.addChild(childCA, RelationshipTypeEnum.ISA); - myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs); - return codeSystem; - } - - private CodeSystem createExternalCsLarge() { - CodeSystem codeSystem = new CodeSystem(); - codeSystem.setUrl(URL_MY_CODE_SYSTEM); - codeSystem.setContent(CodeSystemContentMode.NOTPRESENT); - IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); - - ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new); - - TermCodeSystemVersion cs = new TermCodeSystemVersion(); - cs.setResource(table); - - TermConcept parentA = new TermConcept(cs, "codeA").setDisplay("CodeA"); - cs.getConcepts().add(parentA); - - for (int i = 0; i < 450; i++) { - TermConcept childI = new TermConcept(cs, "subCodeA"+i).setDisplay("Sub-code A"+i); - parentA.addChild(childI, RelationshipTypeEnum.ISA); - } - - TermConcept parentB = new TermConcept(cs, "codeB").setDisplay("CodeB"); - cs.getConcepts().add(parentB); - - for (int i = 0; i < 450; i++) { - TermConcept childI = new TermConcept(cs, "subCodeB"+i).setDisplay("Sub-code B"+i); - parentB.addChild(childI, RelationshipTypeEnum.ISA); - } - - myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs); + myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs); return codeSystem; } @@ -153,17 +121,48 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { TermConcept goodbye = new TermConcept(cs, "goodbye").setDisplay("Goodbye"); cs.getConcepts().add(goodbye); - + TermConcept dogs = new TermConcept(cs, "dogs").setDisplay("Dogs"); cs.getConcepts().add(dogs); - + TermConcept labrador = new TermConcept(cs, "labrador").setDisplay("Labrador"); dogs.addChild(labrador, RelationshipTypeEnum.ISA); TermConcept beagle = new TermConcept(cs, "beagle").setDisplay("Beagle"); dogs.addChild(beagle, RelationshipTypeEnum.ISA); - myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs); + myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs); + return codeSystem; + } + + private CodeSystem createExternalCsLarge() { + CodeSystem codeSystem = new CodeSystem(); + codeSystem.setUrl(URL_MY_CODE_SYSTEM); + codeSystem.setContent(CodeSystemContentMode.NOTPRESENT); + IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); + + ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new); + + TermCodeSystemVersion cs = new TermCodeSystemVersion(); + cs.setResource(table); + + TermConcept parentA = new TermConcept(cs, "codeA").setDisplay("CodeA"); + cs.getConcepts().add(parentA); + + for (int i = 0; i < 450; i++) { + TermConcept childI = new TermConcept(cs, "subCodeA" + i).setDisplay("Sub-code A" + i); + parentA.addChild(childI, RelationshipTypeEnum.ISA); + } + + TermConcept parentB = new TermConcept(cs, "codeB").setDisplay("CodeB"); + cs.getConcepts().add(parentB); + + for (int i = 0; i < 450; i++) { + TermConcept childI = new TermConcept(cs, "subCodeB" + i).setDisplay("Sub-code B" + i); + parentB.addChild(childI, RelationshipTypeEnum.ISA); + } + + myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs); return codeSystem; } @@ -171,17 +170,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { //@formatter:off CodeSystem codeSystem = new CodeSystem(); codeSystem.setUrl(URL_MY_CODE_SYSTEM); - codeSystem.setContent(CodeSystemContentMode.COMPLETE); + codeSystem.setContent(CodeSystemContentMode.COMPLETE); codeSystem .addConcept().setCode("A").setDisplay("Code A") - .addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA") - .addConcept(new ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA")) - ) - .addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB")); + .addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA") + .addConcept(new ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA")) + ) + .addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB")); codeSystem .addConcept().setCode("B").setDisplay("Code B") - .addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA")) - .addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB")); + .addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA")) + .addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB")); //@formatter:on myCodeSystemDao.create(codeSystem, mySrd); @@ -234,15 +233,15 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { //@formatter:off CodeSystem codeSystem = new CodeSystem(); codeSystem.setUrl(URL_MY_CODE_SYSTEM); - codeSystem.setContent(CodeSystemContentMode.COMPLETE); + codeSystem.setContent(CodeSystemContentMode.COMPLETE); codeSystem .addConcept().setCode("A").setDisplay("Code A") - .addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")) - .addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB")); + .addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")) + .addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB")); codeSystem .addConcept().setCode("B").setDisplay("Code A") - .addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code AA")) - .addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code AB")); + .addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code AA")) + .addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code AB")); //@formatter:on IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); @@ -252,6 +251,20 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } + @Test + public void testConceptTimestamps() { + long start = System.currentTimeMillis() - 10; + + createExternalCsDogs(); + + runInTransaction(() -> { + List concepts = myTermConceptDao.findAll(); + for (TermConcept next : concepts) { + assertTrue(next.getUpdated().getTime() > start); + } + }); + } + @Test public void testExpandInvalid() { createExternalCsAndLocalVs(); @@ -278,17 +291,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { valueSet.setUrl(URL_MY_VALUE_SET); valueSet.getCompose() .addInclude() - .setSystem(codeSystem.getUrl()) - .addConcept(new ConceptReferenceComponent().setCode("hello")) - .addConcept(new ConceptReferenceComponent().setCode("goodbye")); + .setSystem(codeSystem.getUrl()) + .addConcept(new ConceptReferenceComponent().setCode("hello")) + .addConcept(new ConceptReferenceComponent().setCode("goodbye")); valueSet.getCompose() .addInclude() - .setSystem(codeSystem.getUrl()) - .addFilter() - .setProperty("concept") - .setOp(FilterOperator.ISA) - .setValue("dogs"); - + .setSystem(codeSystem.getUrl()) + .addFilter() + .setProperty("concept") + .setOp(FilterOperator.ISA) + .setValue("dogs"); + myValueSetDao.create(valueSet, mySrd); ValueSet result = myValueSetDao.expand(valueSet, ""); @@ -300,47 +313,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } - // TODO: get this working - @Ignore - @Test - public void testExpandWithOpEquals() { - - - ValueSet result = myValueSetDao.expandByIdentifier("http://hl7.org/fhir/ValueSet/doc-typecodes", ""); - ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result)); - } - - - @Test - public void testExpandWithCodesAndDisplayFilterPartialOnFilter() { - CodeSystem codeSystem = createExternalCsDogs(); - - ValueSet valueSet = new ValueSet(); - valueSet.setUrl(URL_MY_VALUE_SET); - valueSet.getCompose() - .addInclude() - .setSystem(codeSystem.getUrl()) - .addConcept(new ConceptReferenceComponent().setCode("hello")) - .addConcept(new ConceptReferenceComponent().setCode("goodbye")); - valueSet.getCompose() - .addInclude() - .setSystem(codeSystem.getUrl()) - .addFilter() - .setProperty("concept") - .setOp(FilterOperator.ISA) - .setValue("dogs"); - - myValueSetDao.create(valueSet, mySrd); - - ValueSet result = myValueSetDao.expand(valueSet, "lab"); - logAndValidateValueSet(result); - - assertEquals(1, result.getExpansion().getTotal()); - ArrayList codes = toCodesContains(result.getExpansion().getContains()); - assertThat(codes, containsInAnyOrder("labrador")); - - } - @Test public void testExpandWithCodesAndDisplayFilterPartialOnCodes() { CodeSystem codeSystem = createExternalCsDogs(); @@ -349,17 +321,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { valueSet.setUrl(URL_MY_VALUE_SET); valueSet.getCompose() .addInclude() - .setSystem(codeSystem.getUrl()) - .addConcept(new ConceptReferenceComponent().setCode("hello")) - .addConcept(new ConceptReferenceComponent().setCode("goodbye")); + .setSystem(codeSystem.getUrl()) + .addConcept(new ConceptReferenceComponent().setCode("hello")) + .addConcept(new ConceptReferenceComponent().setCode("goodbye")); valueSet.getCompose() .addInclude() - .setSystem(codeSystem.getUrl()) - .addFilter() - .setProperty("concept") - .setOp(FilterOperator.ISA) - .setValue("dogs"); - + .setSystem(codeSystem.getUrl()) + .addFilter() + .setProperty("concept") + .setOp(FilterOperator.ISA) + .setValue("dogs"); + myValueSetDao.create(valueSet, mySrd); ValueSet result = myValueSetDao.expand(valueSet, "hel"); @@ -389,6 +361,36 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } + @Test + public void testExpandWithCodesAndDisplayFilterPartialOnFilter() { + CodeSystem codeSystem = createExternalCsDogs(); + + ValueSet valueSet = new ValueSet(); + valueSet.setUrl(URL_MY_VALUE_SET); + valueSet.getCompose() + .addInclude() + .setSystem(codeSystem.getUrl()) + .addConcept(new ConceptReferenceComponent().setCode("hello")) + .addConcept(new ConceptReferenceComponent().setCode("goodbye")); + valueSet.getCompose() + .addInclude() + .setSystem(codeSystem.getUrl()) + .addFilter() + .setProperty("concept") + .setOp(FilterOperator.ISA) + .setValue("dogs"); + + myValueSetDao.create(valueSet, mySrd); + + ValueSet result = myValueSetDao.expand(valueSet, "lab"); + logAndValidateValueSet(result); + + assertEquals(1, result.getExpansion().getTotal()); + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, containsInAnyOrder("labrador")); + + } + @Test public void testExpandWithDisplayInExternalValueSetFuzzyMatching() { createExternalCsAndLocalVs(); @@ -484,7 +486,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { @Test public void testExpandWithIsAInExternalValueSetReindex() { BaseHapiTerminologySvcImpl.setForceSaveDeferredAlwaysForUnitTest(true); - + createExternalCsAndLocalVs(); mySystemDao.markAllResourcesForReindexing(); @@ -494,7 +496,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { myHapiTerminologySvc.saveDeferred(); myHapiTerminologySvc.saveDeferred(); myHapiTerminologySvc.saveDeferred(); - + ValueSet vs = new ValueSet(); ConceptSetComponent include = vs.getCompose().addInclude(); include.setSystem(URL_MY_CODE_SYSTEM); @@ -542,7 +544,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { assertEquals("unable to find code system http://example.com/my_code_systemAA", e.getMessage()); } } - + + // TODO: get this working + @Ignore + @Test + public void testExpandWithOpEquals() { + + + ValueSet result = myValueSetDao.expandByIdentifier("http://hl7.org/fhir/ValueSet/doc-typecodes", ""); + ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result)); + } + @Test public void testExpandWithSystemAndCodesAndFilterKeywordInLocalValueSet() { createLocalCsAndVs(); @@ -555,12 +567,12 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue("AAA"); ValueSet result = myValueSetDao.expand(vs, null); - + // Technically it's not valid to expand a ValueSet with both includes and filters so the // result fails validation because of the input.. we're being permissive by allowing both // though, so we won't validate the input result.setCompose(new ValueSetComposeComponent()); - + logAndValidateValueSet(result); ArrayList codes = toCodesContains(result.getExpansion().getContains()); @@ -631,7 +643,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { valueSet.setUrl(URL_MY_VALUE_SET); valueSet.getCompose() .addInclude() - .setSystem(codeSystem.getUrl()); + .setSystem(codeSystem.getUrl()); ValueSet result = myValueSetDao.expand(valueSet, ""); logAndValidateValueSet(result); @@ -713,7 +725,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { cs.setResource(table); TermConcept parentA = new TermConcept(cs, "ParentA").setDisplay("Parent A"); cs.getConcepts().add(parentA); - myTermSvc.storeNewCodeSystemVersion(table.getId(), "http://snomed.info/sct","Snomed CT" , cs); + myTermSvc.storeNewCodeSystemVersion(table.getId(), "http://snomed.info/sct", "Snomed CT", cs); StringType code = new StringType("ParentA"); StringType system = new StringType("http://snomed.info/sct"); @@ -769,7 +781,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { myTermSvc.saveDeferred(); mySystemDao.performReindexingPass(null); myTermSvc.saveDeferred(); - + // Again mySystemDao.markAllResourcesForReindexing(); mySystemDao.performReindexingPass(null); @@ -818,19 +830,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } - @Test - public void testSearchCodeInUnknownCodeSystem() { - - SearchParameterMap params = new SearchParameterMap(); - - try { - params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN)); - assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); - } catch (InvalidRequestException e) { - assertEquals("Unable to find imported value set http://example.com/my_value_set", e.getMessage()); - } - } - @Test public void testSearchCodeBelowBuiltInCodesystem() { AllergyIntolerance ai1 = new AllergyIntolerance(); @@ -918,33 +917,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } - - @Test - public void testSearchCodeBelowLocalCodesystem() { - createLocalCsAndVs(); - - Observation obsAA = new Observation(); - obsAA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("AA"); - IIdType idAA = myObservationDao.create(obsAA, mySrd).getId().toUnqualifiedVersionless(); - - Observation obsBA = new Observation(); - obsBA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("BA"); - IIdType idBA = myObservationDao.create(obsBA, mySrd).getId().toUnqualifiedVersionless(); - - Observation obsCA = new Observation(); - obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA"); - IIdType idCA = myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless(); - - SearchParameterMap params = new SearchParameterMap(); - params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "A").setModifier(TokenParamModifier.BELOW)); - assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue())); - - params = new SearchParameterMap(); - params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "AAA").setModifier(TokenParamModifier.BELOW)); - assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); - - } - @Test public void testSearchCodeBelowExternalCodesystemLarge() { createExternalCsLarge(); @@ -975,6 +947,32 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } + @Test + public void testSearchCodeBelowLocalCodesystem() { + createLocalCsAndVs(); + + Observation obsAA = new Observation(); + obsAA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("AA"); + IIdType idAA = myObservationDao.create(obsAA, mySrd).getId().toUnqualifiedVersionless(); + + Observation obsBA = new Observation(); + obsBA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("BA"); + IIdType idBA = myObservationDao.create(obsBA, mySrd).getId().toUnqualifiedVersionless(); + + Observation obsCA = new Observation(); + obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA"); + IIdType idCA = myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless(); + + SearchParameterMap params = new SearchParameterMap(); + params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "A").setModifier(TokenParamModifier.BELOW)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue())); + + params = new SearchParameterMap(); + params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "AAA").setModifier(TokenParamModifier.BELOW)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); + + } + @Test public void testSearchCodeInBuiltInValueSet() { AllergyIntolerance ai1 = new AllergyIntolerance(); @@ -1019,7 +1017,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { SearchParameterMap params; ourLog.info("testSearchCodeInEmptyValueSet without status"); - + params = new SearchParameterMap(); params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN)); assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); @@ -1030,7 +1028,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN)); params.add(Observation.SP_STATUS, new TokenParam(null, "final")); assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); - + ourLog.info("testSearchCodeInEmptyValueSet done"); } @@ -1093,7 +1091,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { assertThat(toUnqualifiedVersionlessIdValues(myAuditEventDao.search(params)), empty()); } - @Test public void testSearchCodeInLocalCodesystem() { createLocalCsAndVs(); @@ -1116,6 +1113,19 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { } + @Test + public void testSearchCodeInUnknownCodeSystem() { + + SearchParameterMap params = new SearchParameterMap(); + + try { + params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN)); + assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty()); + } catch (InvalidRequestException e) { + assertEquals("Unable to find imported value set http://example.com/my_value_set", e.getMessage()); + } + } + @Test public void testSearchCodeInValueSetThatImportsInvalidCodeSystem() { ValueSet valueSet = new ValueSet(); @@ -1126,12 +1136,12 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test { SearchParameterMap params; ourLog.info("testSearchCodeInEmptyValueSet without status"); - + params = new SearchParameterMap(); params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN)); try { myObservationDao.search(params); - } catch(InvalidRequestException e) { + } catch (InvalidRequestException e) { assertEquals("Unable to expand imported value set: Unable to find imported value set http://non_existant_VS", e.getMessage()); }