From 0c955f7f27057fa49e869bc7d835099ffa5ba2d4 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Sun, 15 Apr 2018 17:20:36 -0400 Subject: [PATCH] More loinc updates --- .../ca/uhn/fhir/jpa/entity/TermConcept.java | 3 ++ .../jpa/term/BaseHapiTerminologySvcImpl.java | 18 +++++----- .../jpa/term/TerminologyLoaderSvcImpl.java | 2 +- .../term/loinc/LoincAnswerListHandler.java | 33 ++++++++----------- .../fhir/jpa/term/loinc/LoincPartHandler.java | 15 +++++---- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java index 96f2efb1109..a329c09df07 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java @@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.entity; import ca.uhn.fhir.context.support.IContextValidationSupport; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; import ca.uhn.fhir.jpa.search.DeferConceptIndexingInterceptor; +import ca.uhn.fhir.util.ValidateUtil; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -10,6 +11,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.hibernate.search.annotations.*; import org.hl7.fhir.r4.model.Coding; +import org.springframework.validation.ValidationUtils; import javax.annotation.Nonnull; import javax.persistence.*; @@ -185,6 +187,7 @@ public class TermConcept implements Serializable { } public void setCode(String theCode) { + ValidateUtil.isNotBlankOrThrowInvalidRequest(theCode, "Code must not be null or empty"); myCode = theCode; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index b837e22d30d..57d6f8c7723 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -432,7 +432,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc fetchParents(concept, retVal); - ourLog.info("Fetched {} codes above code {} in {}ms", new Object[] {retVal.size(), theCode, stopwatch.getMillis()}); + ourLog.info("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis()); return retVal; } @@ -463,7 +463,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc fetchChildren(concept, retVal); - ourLog.info("Fetched {} codes below code {} in {}ms", new Object[] {retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS)}); + ourLog.info("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS)); return retVal; } @@ -537,7 +537,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc } private void processDeferredConceptMaps() { - int count = Math.min(myDeferredConceptMaps.size(), 5); + int count = Math.min(myDeferredConceptMaps.size(), 20); for (ConceptMap nextConceptMap : new ArrayList<>(myDeferredConceptMaps.subList(0, count))) { ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId()); createOrUpdateConceptMap(nextConceptMap); @@ -559,7 +559,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc if (codeCount > 0) { ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)", - new Object[] {codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)}); + codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)); } if (codeCount == 0) { @@ -580,7 +580,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc if (relCount > 0) { ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)", - new Object[] {relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)}); + relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount)); } if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) { @@ -589,13 +589,13 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc } private void processDeferredValueSets() { - int count = Math.min(myDeferredValueSets.size(), 5); + int count = Math.min(myDeferredValueSets.size(), 20); for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) { ourLog.info("Creating ValueSet: {}", nextValueSet.getId()); createOrUpdateValueSet(nextValueSet); myDeferredValueSets.remove(nextValueSet); } - ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredConceptMaps.size()); + ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredValueSets.size()); } private void processReindexing() { @@ -668,7 +668,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc count++; } - ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] {count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count)}); + ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count)); } }); @@ -758,7 +758,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc // Grab the existing versions so we can delete them later List existing = myCodeSystemVersionDao.findByCodeSystemResource(theCodeSystemResourcePid); - verifyNoDuplicates(theCodeSystemVersion.getConcepts(), new HashSet()); +// verifyNoDuplicates(theCodeSystemVersion.getConcepts(), new HashSet()); /* * For now we always delete old versions.. At some point it would be nice to allow configuration to keep old versions diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java index c2d9673ae95..a16289f4c56 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java @@ -240,7 +240,7 @@ public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc { iterateOverZipFile(theDescriptors, LOINC_HIERARCHY_FILE, handler, ',', QuoteMode.NON_NUMERIC); // Answer lists (ValueSets of potential answers/values for loinc "questions") - handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, propertyNames, valueSets); + handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, propertyNames, valueSets, conceptMaps); iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_FILE, handler, ',', QuoteMode.NON_NUMERIC); // Answer list links (connects loinc observation codes to answerlist codes) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincAnswerListHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincAnswerListHandler.java index a7772664ba6..611b5ea7570 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincAnswerListHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincAnswerListHandler.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term.loinc; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,9 +23,8 @@ package ca.uhn.fhir.jpa.term.loinc; import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc; -import ca.uhn.fhir.jpa.term.IRecordHandler; import org.apache.commons.csv.CSVRecord; -import org.hl7.fhir.r4.model.Enumerations; +import org.hl7.fhir.r4.model.ConceptMap; import org.hl7.fhir.r4.model.ValueSet; import java.util.HashMap; @@ -33,10 +32,9 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.StringUtils.trim; +import static org.apache.commons.lang3.StringUtils.*; -public class LoincAnswerListHandler implements IRecordHandler { +public class LoincAnswerListHandler extends BaseHandler { private final Map myCode2Concept; private final TermCodeSystemVersion myCodeSystemVersion; @@ -44,7 +42,8 @@ public class LoincAnswerListHandler implements IRecordHandler { private final List myValueSets; private final Map myIdToValueSet = new HashMap<>(); - public LoincAnswerListHandler(TermCodeSystemVersion theCodeSystemVersion, Map theCode2concept, Set thePropertyNames, List theValueSets) { + public LoincAnswerListHandler(TermCodeSystemVersion theCodeSystemVersion, Map theCode2concept, Set thePropertyNames, List theValueSets, List theConceptMaps) { + super(theCode2concept, theValueSets, theConceptMaps); myCodeSystemVersion = theCodeSystemVersion; myCode2Concept = theCode2concept; myPropertyNames = thePropertyNames; @@ -70,6 +69,10 @@ public class LoincAnswerListHandler implements IRecordHandler { String extCodeSystem = trim(theRecord.get("ExtCodeSystem")); String extCodeSystemVersion = trim(theRecord.get("ExtCodeSystemVersion")); + if (isBlank(answerString)) { + return; + } + // Answer list code if (!myCode2Concept.containsKey(answerListId)) { TermConcept concept = new TermConcept(myCodeSystemVersion, answerListId); @@ -88,21 +91,13 @@ public class LoincAnswerListHandler implements IRecordHandler { } // Answer list ValueSet - ValueSet vs; - if (!myIdToValueSet.containsKey(answerListId)) { - vs = new ValueSet(); - vs.setUrl("urn:oid:" + answerListOid); + ValueSet vs = getValueSet(answerListId, "urn:oid:" + answerListOid, answerListName); + if (vs.getIdentifier().isEmpty()) { vs.addIdentifier() .setSystem(IHapiTerminologyLoaderSvc.LOINC_URI) .setValue(answerListId); - vs.setId(answerListId); - vs.setName(answerListName); - vs.setStatus(Enumerations.PublicationStatus.ACTIVE); - myIdToValueSet.put(answerListId, vs); - myValueSets.add(vs); - } else { - vs = myIdToValueSet.get(answerListId); } + vs .getCompose() .getIncludeFirstRep() diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartHandler.java index 0b1600aa143..59af692d285 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartHandler.java @@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term.loinc; * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,9 +58,13 @@ public class LoincPartHandler implements IRecordHandler { // return; // } - TermConcept concept = new TermConcept(myCodeSystemVersion, partNumber); - concept.setDisplay(partName); - + TermConcept concept = myCode2Concept.get(partNumber); + if (concept == null) { + concept = new TermConcept(myCodeSystemVersion, partNumber); + concept.setDisplay(partName); + myCode2Concept.put(partNumber, concept); + } + if (isNotBlank(partDisplayName)) { concept.addDesignation() .setConcept(concept) @@ -68,7 +72,6 @@ public class LoincPartHandler implements IRecordHandler { .setValue(partDisplayName); } - myCode2Concept.put(partDisplayName, concept); } }