honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'loinc_loader_update'

This commit is contained in:
jamesagnew 2018-04-15 17:43:32 -04:00
parent c450d51440 13242d77ed
commit ecc23732d3
5 changed files with 48 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java
View File

@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.entity;
import ca.uhn.fhir.context.support.IContextValidationSupport;
import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
import ca.uhn.fhir.jpa.search.DeferConceptIndexingInterceptor;
import ca.uhn.fhir.util.ValidateUtil;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
@ -10,6 +11,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.hibernate.search.annotations.*;
import org.hl7.fhir.r4.model.Coding;
import org.springframework.validation.ValidationUtils;
import javax.annotation.Nonnull;
import javax.persistence.*;
@ -185,6 +187,7 @@ public class TermConcept implements Serializable {
}
public void setCode(String theCode) {
ValidateUtil.isNotBlankOrThrowInvalidRequest(theCode, "Code must not be null or empty");
myCode = theCode;
}

31
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -432,7 +432,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
fetchParents(concept, retVal);
ourLog.info("Fetched {} codes above code {} in {}ms", new Object[] {retVal.size(), theCode, stopwatch.getMillis()});
ourLog.info("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
return retVal;
}
@ -463,7 +463,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
fetchChildren(concept, retVal);
ourLog.info("Fetched {} codes below code {} in {}ms", new Object[] {retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS)});
ourLog.info("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
return retVal;
}
@ -537,7 +537,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
}
private void processDeferredConceptMaps() {
int count = Math.min(myDeferredConceptMaps.size(), 5);
int count = Math.min(myDeferredConceptMaps.size(), 20);
for (ConceptMap nextConceptMap : new ArrayList<>(myDeferredConceptMaps.subList(0, count))) {
ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
createOrUpdateConceptMap(nextConceptMap);
@ -559,7 +559,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
if (codeCount > 0) {
ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
new Object[] {codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
}
if (codeCount == 0) {
@ -580,7 +580,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
if (relCount > 0) {
ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)",
new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)});
relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
}
if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {
@ -589,13 +589,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
}
private void processDeferredValueSets() {
int count = Math.min(myDeferredValueSets.size(), 5);
int count = Math.min(myDeferredValueSets.size(), 20);
for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) {
ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
createOrUpdateValueSet(nextValueSet);
myDeferredValueSets.remove(nextValueSet);
}
ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredConceptMaps.size());
ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredValueSets.size());
}
private void processReindexing() {
@ -668,7 +668,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
count++;
}
ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] {count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count)});
ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count));
}
});
@ -758,6 +758,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
// Grab the existing versions so we can delete them later
List<TermCodeSystemVersion> existing = myCodeSystemVersionDao.findByCodeSystemResource(theCodeSystemResourcePid);
// verifyNoDuplicates(theCodeSystemVersion.getConcepts(), new HashSet<String>());
/*
* For now we always delete old versions.. At some point it would be nice to allow configuration to keep old versions
*/
@ -909,6 +911,15 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc
return retVal;
}
private void verifyNoDuplicates(Collection<TermConcept> theConcepts, Set<String> theCodes) {
for (TermConcept next : theConcepts) {
if (!theCodes.add(next.getCode())) {
throw new InvalidRequestException("Duplicate code " + next.getCode() + " found in codesystem after checking " + theCodes.size() + " codes");
}
verifyNoDuplicates(next.getChildren().stream().map(TermConceptParentChildLink::getChild).collect(Collectors.toList()), theCodes);
}
}
/**
* This method is present only for unit tests, do not call from client code
*/

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java
View File

@ -240,7 +240,7 @@ public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
iterateOverZipFile(theDescriptors, LOINC_HIERARCHY_FILE, handler, ',', QuoteMode.NON_NUMERIC);
// Answer lists (ValueSets of potential answers/values for loinc "questions")
handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, propertyNames, valueSets);
handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, propertyNames, valueSets, conceptMaps);
iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_FILE, handler, ',', QuoteMode.NON_NUMERIC);
// Answer list links (connects loinc observation codes to answerlist codes)

33
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincAnswerListHandler.java
View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term.loinc;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -23,9 +23,8 @@ package ca.uhn.fhir.jpa.term.loinc;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc;
import ca.uhn.fhir.jpa.term.IRecordHandler;
import org.apache.commons.csv.CSVRecord;
import org.hl7.fhir.r4.model.Enumerations;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.HashMap;
@ -33,10 +32,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.trim;
import static org.apache.commons.lang3.StringUtils.*;
public class LoincAnswerListHandler implements IRecordHandler {
public class LoincAnswerListHandler extends BaseHandler {
private final Map<String, TermConcept> myCode2Concept;
private final TermCodeSystemVersion myCodeSystemVersion;
@ -44,7 +42,8 @@ public class LoincAnswerListHandler implements IRecordHandler {
private final List<ValueSet> myValueSets;
private final Map<String, ValueSet> myIdToValueSet = new HashMap<>();
public LoincAnswerListHandler(TermCodeSystemVersion theCodeSystemVersion, Map<String, TermConcept> theCode2concept, Set<String> thePropertyNames, List<ValueSet> theValueSets) {
public LoincAnswerListHandler(TermCodeSystemVersion theCodeSystemVersion, Map<String, TermConcept> theCode2concept, Set<String> thePropertyNames, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
super(theCode2concept, theValueSets, theConceptMaps);
myCodeSystemVersion = theCodeSystemVersion;
myCode2Concept = theCode2concept;
myPropertyNames = thePropertyNames;
@ -70,6 +69,10 @@ public class LoincAnswerListHandler implements IRecordHandler {
String extCodeSystem = trim(theRecord.get("ExtCodeSystem"));
String extCodeSystemVersion = trim(theRecord.get("ExtCodeSystemVersion"));
if (isBlank(answerString)) {
return;
}
// Answer list code
if (!myCode2Concept.containsKey(answerListId)) {
TermConcept concept = new TermConcept(myCodeSystemVersion, answerListId);
@ -88,21 +91,13 @@ public class LoincAnswerListHandler implements IRecordHandler {
}
// Answer list ValueSet
ValueSet vs;
if (!myIdToValueSet.containsKey(answerListId)) {
vs = new ValueSet();
vs.setUrl("urn:oid:" + answerListOid);
ValueSet vs = getValueSet(answerListId, "urn:oid:" + answerListOid, answerListName);
if (vs.getIdentifier().isEmpty()) {
vs.addIdentifier()
.setSystem(IHapiTerminologyLoaderSvc.LOINC_URI)
.setValue(answerListId);
vs.setId(answerListId);
vs.setName(answerListName);
vs.setStatus(Enumerations.PublicationStatus.ACTIVE);
myIdToValueSet.put(answerListId, vs);
myValueSets.add(vs);
} else {
vs = myIdToValueSet.get(answerListId);
}
vs
.getCompose()
.getIncludeFirstRep()

15
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartHandler.java
View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term.loinc;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -58,9 +58,13 @@ public class LoincPartHandler implements IRecordHandler {
// return;
// }
TermConcept concept = new TermConcept(myCodeSystemVersion, partNumber);
concept.setDisplay(partName);
TermConcept concept = myCode2Concept.get(partNumber);
if (concept == null) {
concept = new TermConcept(myCodeSystemVersion, partNumber);
concept.setDisplay(partName);
myCode2Concept.put(partNumber, concept);
}
if (isNotBlank(partDisplayName)) {
concept.addDesignation()
.setConcept(concept)
@ -68,7 +72,6 @@ public class LoincPartHandler implements IRecordHandler {
.setValue(partDisplayName);
}
myCode2Concept.put(partDisplayName, concept);
}
}