honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean up concept reindexing

This commit is contained in:
jamesagnew 2018-07-15 17:18:07 -04:00
parent 116b9e335d
commit 401a4cd738
8 changed files with 272 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java
View File

@ -39,9 +39,9 @@ import java.util.Map.Entry;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -92,9 +92,9 @@ import static org.apache.commons.lang3.StringUtils.*;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

36
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptDao.java
View File

@ -10,6 +10,7 @@ import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.util.Date;
import java.util.List;
/*
@ -21,9 +22,9 @@ import java.util.List;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -34,21 +35,42 @@ import java.util.List;
public interface ITermConceptDao extends JpaRepository<TermConcept, Long> {
@Query("SELECT COUNT(t) FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid")
Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid);
/**
* Used in Smile CDR - Do not delete
*/
@SuppressWarnings("unused")
@Query("SELECT COUNT(*) FROM TermConcept t WHERE t.myUpdated > :cutoff")
long countUpdatedAfter(@Param("cutoff") Date theFullTextIndexedUntil);
/**
* Used in Smile CDR - Do not delete
*/
@SuppressWarnings("unused")
@Query("SELECT t FROM TermConcept t ORDER BY t.myUpdated ASC")
Slice<TermConcept> findAllOrderedByLastUpdated(Pageable thePage);
@Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system AND c.myCode = :code")
TermConcept findByCodeSystemAndCode(@Param("code_system") TermCodeSystemVersion theCodeSystem, @Param("code") String theCode);
@Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system")
List<TermConcept> findByCodeSystemVersion(@Param("code_system") TermCodeSystemVersion theCodeSystem);
@Query("SELECT t FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid")
Slice<TermConcept> findByCodeSystemVersion(Pageable thePage, @Param("cs_pid") Long thePid);
@Query("SELECT COUNT(t) FROM TermConcept t WHERE t.myCodeSystem.myId = :cs_pid")
Integer countByCodeSystemVersion(@Param("cs_pid") Long thePid);
@Query("SELECT c FROM TermConcept c WHERE c.myCodeSystem = :code_system")
List<TermConcept> findByCodeSystemVersion(@Param("code_system") TermCodeSystemVersion theCodeSystem);
@Query("SELECT t FROM TermConcept t WHERE t.myIndexStatus = null")
Page<TermConcept> findResourcesRequiringReindexing(Pageable thePageRequest);
/**
* Used in Smile CDR - Do not delete
*/
@SuppressWarnings("unused")
@Query("SELECT t FROM TermConcept t WHERE t.myUpdated > :cutoff ORDER BY t.myUpdated ASC")
Slice<TermConcept> findUpdatedAfterOrderedByLastUpdated(Pageable thePage, @Param("cutoff") Date theFullTextIndexedUntil);
@Query("UPDATE TermConcept t SET t.myIndexStatus = null")
@Modifying
int markAllForReindexing();

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java
View File

@ -18,9 +18,9 @@ import java.util.Collection;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

25
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java
View File

@ -45,7 +45,8 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
@Table(name = "TRM_CONCEPT", uniqueConstraints = {
@UniqueConstraint(name = "IDX_CONCEPT_CS_CODE", columnNames = {"CODESYSTEM_PID", "CODE"})
}, indexes = {
@Index(name = "IDX_CONCEPT_INDEXSTATUS", columnList = "INDEX_STATUS")
@Index(name = "IDX_CONCEPT_INDEXSTATUS", columnList = "INDEX_STATUS"),
@Index(name = "IDX_CONCEPT_UPDATED", columnList = "CONCEPT_UPDATED")
})
public class TermConcept implements Serializable {
protected static final int MAX_DESC_LENGTH = 400;
@ -59,15 +60,15 @@ public class TermConcept implements Serializable {
@Column(name = "CODE", length = 100, nullable = false)
@Fields({@Field(name = "myCode", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "exactAnalyzer")),})
private String myCode;
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "CONCEPT_UPDATED", nullable = true)
private Date myUpdated;
@ManyToOne()
@JoinColumn(name = "CODESYSTEM_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_CONCEPT_PID_CS_PID"))
private TermCodeSystemVersion myCodeSystem;
@Column(name = "CODESYSTEM_PID", insertable = false, updatable = false)
@Fields({@Field(name = "myCodeSystemVersionPid")})
private long myCodeSystemVersionPid;
@Column(name = "DISPLAY", length = MAX_DESC_LENGTH, nullable = true)
@Fields({
@Field(name = "myDisplay", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "standardAnalyzer")),
@ -76,15 +77,12 @@ public class TermConcept implements Serializable {
@Field(name = "myDisplayPhonetic", index = org.hibernate.search.annotations.Index.YES, store = Store.NO, analyze = Analyze.YES, analyzer = @Analyzer(definition = "autocompletePhoneticAnalyzer"))
})
private String myDisplay;
@OneToMany(mappedBy = "myConcept", orphanRemoval = false)
@Field(name = "PROPmyProperties", analyzer = @Analyzer(definition = "termConceptPropertyAnalyzer"))
@FieldBridge(impl = TermConceptPropertyFieldBridge.class)
private Collection<TermConceptProperty> myProperties;
@OneToMany(mappedBy = "myConcept", orphanRemoval = false)
private Collection<TermConceptDesignation> myDesignations;
@Id()
@SequenceGenerator(name = "SEQ_CONCEPT_PID", sequenceName = "SEQ_CONCEPT_PID")
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CONCEPT_PID")
@ -92,18 +90,17 @@ public class TermConcept implements Serializable {
private Long myId;
@Column(name = "INDEX_STATUS", nullable = true)
private Long myIndexStatus;
@Transient
@Field(name = "myParentPids", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "conceptParentPidsAnalyzer"))
@Lob
@Column(name="PARENT_PIDS", nullable = true)
private String myParentPids;
@OneToMany(cascade = {}, fetch = FetchType.LAZY, mappedBy = "myChild")
private Collection<TermConceptParentChildLink> myParents;
@Column(name = "CODE_SEQUENCE", nullable = true)
private Integer mySequence;
public TermConcept() {
super();
}
public TermConcept(TermCodeSystemVersion theCs, String theCode) {
setCodeSystemVersion(theCs);
setCode(theCode);
@ -296,6 +293,14 @@ public class TermConcept implements Serializable {
return null;
}
public Date getUpdated() {
return myUpdated;
}
public void setUpdated(Date theUpdated) {
myUpdated = theUpdated;
}
@Override
public int hashCode() {
HashCodeBuilder b = new HashCodeBuilder();

50
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -327,26 +327,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
private <T> void doDelete(String theDescriptor, Supplier<Slice<T>> theLoader, Supplier<Integer> theCounter, JpaRepository<T, ?> theDao) {
int count;
ourLog.info(" * Deleting {}", theDescriptor);
int totalCount = theCounter.get();
StopWatch sw = new StopWatch();
count = 0;
while (true) {
Slice<T> link = theLoader.get();
if (link.hasContent() == false) {
break;
}
theDao.deleteInBatch(link);
count += link.getNumberOfElements();
ourLog.info(" * {} {} deleted - {}/sec - ETA: {}", count, theDescriptor, sw.formatThroughput(count, TimeUnit.SECONDS), sw.getEstimatedTimeRemaining(count, totalCount));
}
theDao.flush();
}
public void deleteConceptMap(ResourceTable theResourceTable) {
// Get existing entity so it can be deleted.
Optional<TermConceptMap> optionalExistingTermConceptMapById = myConceptMapDao.findTermConceptMapByResourcePid(theResourceTable.getId());
@ -388,6 +368,26 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
deleteConceptMap(theResourceTable);
}
private <T> void doDelete(String theDescriptor, Supplier<Slice<T>> theLoader, Supplier<Integer> theCounter, JpaRepository<T, ?> theDao) {
int count;
ourLog.info(" * Deleting {}", theDescriptor);
int totalCount = theCounter.get();
StopWatch sw = new StopWatch();
count = 0;
while (true) {
Slice<T> link = theLoader.get();
if (link.hasContent() == false) {
break;
}
theDao.deleteInBatch(link);
count += link.getNumberOfElements();
ourLog.info(" * {} {} deleted - {}/sec - ETA: {}", count, theDescriptor, sw.formatThroughput(count, TimeUnit.SECONDS), sw.getEstimatedTimeRemaining(count, totalCount));
}
theDao.flush();
}
private int ensureParentsSaved(Collection<TermConceptParentChildLink> theParents) {
ourLog.trace("Checking {} parents", theParents.size());
int retVal = 0;
@ -397,6 +397,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
TermConcept nextParent = nextLink.getParent();
retVal += ensureParentsSaved(nextParent.getParents());
if (nextParent.getId() == null) {
nextParent.setUpdated(new Date());
myConceptDao.saveAndFlush(nextParent);
retVal++;
ourLog.debug("Saved parent code {} and got id {}", nextParent.getCode(), nextParent.getId());
@ -887,9 +888,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
for (TermConcept nextConcept : concepts) {
StringBuilder parentsBuilder = new StringBuilder();
createParentsString(parentsBuilder, nextConcept.getId());
nextConcept.setParentPids(parentsBuilder.toString());
if (isBlank(nextConcept.getParentPidsAsString())) {
StringBuilder parentsBuilder = new StringBuilder();
createParentsString(parentsBuilder, nextConcept.getId());
nextConcept.setParentPids(parentsBuilder.toString());
}
saveConcept(nextConcept);
count++;
@ -923,6 +926,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
if (theConcept.getId() == null || theConcept.getIndexStatus() == null) {
retVal++;
theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);
theConcept.setUpdated(new Date());
myConceptDao.save(theConcept);
for (TermConceptProperty next : theConcept.getProperties()) {

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java
View File

@ -104,6 +104,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest {
@Qualifier("myConceptMapDaoR4")
protected IFhirResourceDaoConceptMap<ConceptMap> myConceptMapDao;
@Autowired
protected ITermConceptDao myTermConceptDao;
@Autowired
@Qualifier("myConditionDaoR4")
protected IFhirResourceDao<Condition> myConditionDao;
@Autowired

376
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyTest.java
View File

@ -1,30 +1,13 @@
package ca.uhn.fhir.jpa.dao.r4;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsStringIgnoringCase;
import static org.hamcrest.Matchers.empty;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.util.*;
import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl;
import org.hl7.fhir.r4.model.*;
import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceCategory;
import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceClinicalStatus;
import org.hl7.fhir.r4.model.CodeSystem.CodeSystemContentMode;
import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
import org.hl7.fhir.r4.model.ValueSet.*;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.*;
import org.springframework.beans.factory.annotation.Autowired;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult;
import ca.uhn.fhir.jpa.dao.SearchParameterMap;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.param.TokenParam;
@ -34,20 +17,36 @@ import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.util.TestUtil;
import ca.uhn.fhir.validation.FhirValidator;
import ca.uhn.fhir.validation.ValidationResult;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.*;
import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceCategory;
import org.hl7.fhir.r4.model.AllergyIntolerance.AllergyIntoleranceClinicalStatus;
import org.hl7.fhir.r4.model.CodeSystem.CodeSystemContentMode;
import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
import org.hl7.fhir.r4.model.ValueSet.*;
import org.junit.*;
import org.springframework.beans.factory.annotation.Autowired;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4TerminologyTest.class);
public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set";
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4TerminologyTest.class);
@Autowired
private IHapiTerminologySvc myHapiTerminologySvc;
@After
public void after() {
myDaoConfig.setDeferIndexingForCodesystemsOfSize(new DaoConfig().getDeferIndexingForCodesystemsOfSize());
BaseHapiTerminologySvcImpl.setForceSaveDeferredAlwaysForUnitTest(false);
}
@ -96,38 +95,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
TermConcept childCA = new TermConcept(cs, "childCA").setDisplay("Child CA");
parentC.addChild(childCA, RelationshipTypeEnum.ISA);
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs);
return codeSystem;
}
private CodeSystem createExternalCsLarge() {
CodeSystem codeSystem = new CodeSystem();
codeSystem.setUrl(URL_MY_CODE_SYSTEM);
codeSystem.setContent(CodeSystemContentMode.NOTPRESENT);
IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new);
TermCodeSystemVersion cs = new TermCodeSystemVersion();
cs.setResource(table);
TermConcept parentA = new TermConcept(cs, "codeA").setDisplay("CodeA");
cs.getConcepts().add(parentA);
for (int i = 0; i < 450; i++) {
TermConcept childI = new TermConcept(cs, "subCodeA"+i).setDisplay("Sub-code A"+i);
parentA.addChild(childI, RelationshipTypeEnum.ISA);
}
TermConcept parentB = new TermConcept(cs, "codeB").setDisplay("CodeB");
cs.getConcepts().add(parentB);
for (int i = 0; i < 450; i++) {
TermConcept childI = new TermConcept(cs, "subCodeB"+i).setDisplay("Sub-code B"+i);
parentB.addChild(childI, RelationshipTypeEnum.ISA);
}
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs);
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs);
return codeSystem;
}
@ -153,17 +121,48 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
TermConcept goodbye = new TermConcept(cs, "goodbye").setDisplay("Goodbye");
cs.getConcepts().add(goodbye);
TermConcept dogs = new TermConcept(cs, "dogs").setDisplay("Dogs");
cs.getConcepts().add(dogs);
TermConcept labrador = new TermConcept(cs, "labrador").setDisplay("Labrador");
dogs.addChild(labrador, RelationshipTypeEnum.ISA);
TermConcept beagle = new TermConcept(cs, "beagle").setDisplay("Beagle");
dogs.addChild(beagle, RelationshipTypeEnum.ISA);
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM,"SYSTEM NAME" , cs);
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs);
return codeSystem;
}
private CodeSystem createExternalCsLarge() {
CodeSystem codeSystem = new CodeSystem();
codeSystem.setUrl(URL_MY_CODE_SYSTEM);
codeSystem.setContent(CodeSystemContentMode.NOTPRESENT);
IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new);
TermCodeSystemVersion cs = new TermCodeSystemVersion();
cs.setResource(table);
TermConcept parentA = new TermConcept(cs, "codeA").setDisplay("CodeA");
cs.getConcepts().add(parentA);
for (int i = 0; i < 450; i++) {
TermConcept childI = new TermConcept(cs, "subCodeA" + i).setDisplay("Sub-code A" + i);
parentA.addChild(childI, RelationshipTypeEnum.ISA);
}
TermConcept parentB = new TermConcept(cs, "codeB").setDisplay("CodeB");
cs.getConcepts().add(parentB);
for (int i = 0; i < 450; i++) {
TermConcept childI = new TermConcept(cs, "subCodeB" + i).setDisplay("Sub-code B" + i);
parentB.addChild(childI, RelationshipTypeEnum.ISA);
}
myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", cs);
return codeSystem;
}
@ -171,17 +170,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
//@formatter:off
CodeSystem codeSystem = new CodeSystem();
codeSystem.setUrl(URL_MY_CODE_SYSTEM);
codeSystem.setContent(CodeSystemContentMode.COMPLETE);
codeSystem.setContent(CodeSystemContentMode.COMPLETE);
codeSystem
.addConcept().setCode("A").setDisplay("Code A")
.addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")
.addConcept(new ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA"))
)
.addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
.addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA")
.addConcept(new ConceptDefinitionComponent().setCode("AAA").setDisplay("Code AAA"))
)
.addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
codeSystem
.addConcept().setCode("B").setDisplay("Code B")
.addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA"))
.addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB"));
.addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code BA"))
.addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code BB"));
//@formatter:on
myCodeSystemDao.create(codeSystem, mySrd);
@ -234,15 +233,15 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
//@formatter:off
CodeSystem codeSystem = new CodeSystem();
codeSystem.setUrl(URL_MY_CODE_SYSTEM);
codeSystem.setContent(CodeSystemContentMode.COMPLETE);
codeSystem.setContent(CodeSystemContentMode.COMPLETE);
codeSystem
.addConcept().setCode("A").setDisplay("Code A")
.addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA"))
.addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
.addConcept(new ConceptDefinitionComponent().setCode("AA").setDisplay("Code AA"))
.addConcept(new ConceptDefinitionComponent().setCode("AB").setDisplay("Code AB"));
codeSystem
.addConcept().setCode("B").setDisplay("Code A")
.addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code AA"))
.addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code AB"));
.addConcept(new ConceptDefinitionComponent().setCode("BA").setDisplay("Code AA"))
.addConcept(new ConceptDefinitionComponent().setCode("BB").setDisplay("Code AB"));
//@formatter:on
IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
@ -252,6 +251,20 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testConceptTimestamps() {
long start = System.currentTimeMillis() - 10;
createExternalCsDogs();
runInTransaction(() -> {
List<TermConcept> concepts = myTermConceptDao.findAll();
for (TermConcept next : concepts) {
assertTrue(next.getUpdated().getTime() > start);
}
});
}
@Test
public void testExpandInvalid() {
createExternalCsAndLocalVs();
@ -278,17 +291,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
valueSet.setUrl(URL_MY_VALUE_SET);
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
myValueSetDao.create(valueSet, mySrd);
ValueSet result = myValueSetDao.expand(valueSet, "");
@ -300,47 +313,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
// TODO: get this working
@Ignore
@Test
public void testExpandWithOpEquals() {
ValueSet result = myValueSetDao.expandByIdentifier("http://hl7.org/fhir/ValueSet/doc-typecodes", "");
ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result));
}
@Test
public void testExpandWithCodesAndDisplayFilterPartialOnFilter() {
CodeSystem codeSystem = createExternalCsDogs();
ValueSet valueSet = new ValueSet();
valueSet.setUrl(URL_MY_VALUE_SET);
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
myValueSetDao.create(valueSet, mySrd);
ValueSet result = myValueSetDao.expand(valueSet, "lab");
logAndValidateValueSet(result);
assertEquals(1, result.getExpansion().getTotal());
ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("labrador"));
}
@Test
public void testExpandWithCodesAndDisplayFilterPartialOnCodes() {
CodeSystem codeSystem = createExternalCsDogs();
@ -349,17 +321,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
valueSet.setUrl(URL_MY_VALUE_SET);
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
myValueSetDao.create(valueSet, mySrd);
ValueSet result = myValueSetDao.expand(valueSet, "hel");
@ -389,6 +361,36 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testExpandWithCodesAndDisplayFilterPartialOnFilter() {
CodeSystem codeSystem = createExternalCsDogs();
ValueSet valueSet = new ValueSet();
valueSet.setUrl(URL_MY_VALUE_SET);
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addConcept(new ConceptReferenceComponent().setCode("hello"))
.addConcept(new ConceptReferenceComponent().setCode("goodbye"));
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl())
.addFilter()
.setProperty("concept")
.setOp(FilterOperator.ISA)
.setValue("dogs");
myValueSetDao.create(valueSet, mySrd);
ValueSet result = myValueSetDao.expand(valueSet, "lab");
logAndValidateValueSet(result);
assertEquals(1, result.getExpansion().getTotal());
ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("labrador"));
}
@Test
public void testExpandWithDisplayInExternalValueSetFuzzyMatching() {
createExternalCsAndLocalVs();
@ -484,7 +486,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
@Test
public void testExpandWithIsAInExternalValueSetReindex() {
BaseHapiTerminologySvcImpl.setForceSaveDeferredAlwaysForUnitTest(true);
createExternalCsAndLocalVs();
mySystemDao.markAllResourcesForReindexing();
@ -494,7 +496,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
myHapiTerminologySvc.saveDeferred();
myHapiTerminologySvc.saveDeferred();
myHapiTerminologySvc.saveDeferred();
ValueSet vs = new ValueSet();
ConceptSetComponent include = vs.getCompose().addInclude();
include.setSystem(URL_MY_CODE_SYSTEM);
@ -542,7 +544,17 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
assertEquals("unable to find code system http://example.com/my_code_systemAA", e.getMessage());
}
}
// TODO: get this working
@Ignore
@Test
public void testExpandWithOpEquals() {
ValueSet result = myValueSetDao.expandByIdentifier("http://hl7.org/fhir/ValueSet/doc-typecodes", "");
ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result));
}
@Test
public void testExpandWithSystemAndCodesAndFilterKeywordInLocalValueSet() {
createLocalCsAndVs();
@ -555,12 +567,12 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue("AAA");
ValueSet result = myValueSetDao.expand(vs, null);
// Technically it's not valid to expand a ValueSet with both includes and filters so the
// result fails validation because of the input.. we're being permissive by allowing both
// though, so we won't validate the input
result.setCompose(new ValueSetComposeComponent());
logAndValidateValueSet(result);
ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
@ -631,7 +643,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
valueSet.setUrl(URL_MY_VALUE_SET);
valueSet.getCompose()
.addInclude()
.setSystem(codeSystem.getUrl());
.setSystem(codeSystem.getUrl());
ValueSet result = myValueSetDao.expand(valueSet, "");
logAndValidateValueSet(result);
@ -713,7 +725,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
cs.setResource(table);
TermConcept parentA = new TermConcept(cs, "ParentA").setDisplay("Parent A");
cs.getConcepts().add(parentA);
myTermSvc.storeNewCodeSystemVersion(table.getId(), "http://snomed.info/sct","Snomed CT" , cs);
myTermSvc.storeNewCodeSystemVersion(table.getId(), "http://snomed.info/sct", "Snomed CT", cs);
StringType code = new StringType("ParentA");
StringType system = new StringType("http://snomed.info/sct");
@ -769,7 +781,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
myTermSvc.saveDeferred();
mySystemDao.performReindexingPass(null);
myTermSvc.saveDeferred();
// Again
mySystemDao.markAllResourcesForReindexing();
mySystemDao.performReindexingPass(null);
@ -818,19 +830,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testSearchCodeInUnknownCodeSystem() {
SearchParameterMap params = new SearchParameterMap();
try {
params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
} catch (InvalidRequestException e) {
assertEquals("Unable to find imported value set http://example.com/my_value_set", e.getMessage());
}
}
@Test
public void testSearchCodeBelowBuiltInCodesystem() {
AllergyIntolerance ai1 = new AllergyIntolerance();
@ -918,33 +917,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testSearchCodeBelowLocalCodesystem() {
createLocalCsAndVs();
Observation obsAA = new Observation();
obsAA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("AA");
IIdType idAA = myObservationDao.create(obsAA, mySrd).getId().toUnqualifiedVersionless();
Observation obsBA = new Observation();
obsBA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("BA");
IIdType idBA = myObservationDao.create(obsBA, mySrd).getId().toUnqualifiedVersionless();
Observation obsCA = new Observation();
obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA");
IIdType idCA = myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless();
SearchParameterMap params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "A").setModifier(TokenParamModifier.BELOW));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue()));
params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "AAA").setModifier(TokenParamModifier.BELOW));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
}
@Test
public void testSearchCodeBelowExternalCodesystemLarge() {
createExternalCsLarge();
@ -975,6 +947,32 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testSearchCodeBelowLocalCodesystem() {
createLocalCsAndVs();
Observation obsAA = new Observation();
obsAA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("AA");
IIdType idAA = myObservationDao.create(obsAA, mySrd).getId().toUnqualifiedVersionless();
Observation obsBA = new Observation();
obsBA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("BA");
IIdType idBA = myObservationDao.create(obsBA, mySrd).getId().toUnqualifiedVersionless();
Observation obsCA = new Observation();
obsCA.getCode().addCoding().setSystem(URL_MY_CODE_SYSTEM).setCode("CA");
IIdType idCA = myObservationDao.create(obsCA, mySrd).getId().toUnqualifiedVersionless();
SearchParameterMap params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "A").setModifier(TokenParamModifier.BELOW));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), containsInAnyOrder(idAA.getValue()));
params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(URL_MY_CODE_SYSTEM, "AAA").setModifier(TokenParamModifier.BELOW));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
}
@Test
public void testSearchCodeInBuiltInValueSet() {
AllergyIntolerance ai1 = new AllergyIntolerance();
@ -1019,7 +1017,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
SearchParameterMap params;
ourLog.info("testSearchCodeInEmptyValueSet without status");
params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
@ -1030,7 +1028,7 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
params.add(Observation.SP_STATUS, new TokenParam(null, "final"));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
ourLog.info("testSearchCodeInEmptyValueSet done");
}
@ -1093,7 +1091,6 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
assertThat(toUnqualifiedVersionlessIdValues(myAuditEventDao.search(params)), empty());
}
@Test
public void testSearchCodeInLocalCodesystem() {
createLocalCsAndVs();
@ -1116,6 +1113,19 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
}
@Test
public void testSearchCodeInUnknownCodeSystem() {
SearchParameterMap params = new SearchParameterMap();
try {
params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(params)), empty());
} catch (InvalidRequestException e) {
assertEquals("Unable to find imported value set http://example.com/my_value_set", e.getMessage());
}
}
@Test
public void testSearchCodeInValueSetThatImportsInvalidCodeSystem() {
ValueSet valueSet = new ValueSet();
@ -1126,12 +1136,12 @@ public class FhirResourceDaoR4TerminologyTest extends BaseJpaR4Test {
SearchParameterMap params;
ourLog.info("testSearchCodeInEmptyValueSet without status");
params = new SearchParameterMap();
params.add(Observation.SP_CODE, new TokenParam(null, URL_MY_VALUE_SET).setModifier(TokenParamModifier.IN));
try {
myObservationDao.search(params);
} catch(InvalidRequestException e) {
} catch (InvalidRequestException e) {
assertEquals("Unable to expand imported value set: Unable to find imported value set http://non_existant_VS", e.getMessage());
}