Liinc updates

This commit is contained in:
James Agnew 2018-06-22 09:59:54 -04:00
parent cef4596211
commit d46b9b00be
12 changed files with 239 additions and 26 deletions

View File

@ -1081,6 +1081,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
throw new InternalErrorException(fe);
}
myConceptMapDao.save(termConceptMap);
int codesSaved = 0;
if (theConceptMap.hasGroup()) {
TermConceptMapGroup termConceptMapGroup;
@ -1116,7 +1117,12 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
termConceptMapGroupElementTarget.setCode(target.getCode());
termConceptMapGroupElementTarget.setDisplay(target.getDisplay());
termConceptMapGroupElementTarget.setEquivalence(target.getEquivalence());
myConceptMapGroupElementTargetDao.saveAndFlush(termConceptMapGroupElementTarget);
myConceptMapGroupElementTargetDao.save(termConceptMapGroupElementTarget);
if (codesSaved++ % 250 == 0) {
ourLog.info("Have saved {} codes in conceptmap", codesSaved);
myConceptMapGroupElementTargetDao.flush();
}
}
}
}

View File

@ -77,6 +77,9 @@ public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
public static final String LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE = "LoincUniversalLabOrdersValueSet.csv";
public static final String LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV = "LoincIeeeMedicalDeviceCodeMappingTable.csv";
public static final String LOINC_IMAGING_DOCUMENT_CODES_FILE = "ImagingDocumentCodes.csv";
public static final String LOINC_GROUP_FILE = "Group.csv";
public static final String LOINC_GROUP_TERMS_FILE = "GroupLoincTerms.csv";
public static final String LOINC_PARENT_GROUP_FILE = "ParentGroup.csv";
private static final int LOG_INCREMENT = 100000;
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvcImpl.class);
@ -124,7 +127,7 @@ public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
String nextFilename = nextZipBytes.getFilename();
if (nextFilename.contains(theFileNamePart)) {
if (nextFilename.endsWith("/" + theFileNamePart)) {
ourLog.info("Processing file {}", nextFilename);
Reader reader;
@ -308,6 +311,18 @@ public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
handler = new LoincImagingDocumentCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
iterateOverZipFile(theDescriptors, LOINC_IMAGING_DOCUMENT_CODES_FILE, handler, ',', QuoteMode.NON_NUMERIC);
// Group File
handler = new LoincGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
iterateOverZipFile(theDescriptors, LOINC_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC);
// Group Terms File
handler = new LoincGroupTermsFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
iterateOverZipFile(theDescriptors, LOINC_GROUP_TERMS_FILE, handler, ',', QuoteMode.NON_NUMERIC);
// Parent Group File
handler = new LoincParentGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
iterateOverZipFile(theDescriptors, LOINC_PARENT_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC);
IOUtils.closeQuietly(theDescriptors);
for (Entry<String, TermConcept> next : code2concept.entrySet()) {

View File

@ -9,9 +9,9 @@ package ca.uhn.fhir.jpa.term.loinc;
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -52,8 +52,10 @@ public abstract class BaseLoincHandler implements IRecordHandler {
BaseLoincHandler(Map<String, TermConcept> theCode2Concept, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, Properties theUploadProperties) {
myValueSets = theValueSets;
myValueSets.forEach(t -> myIdToValueSet.put(t.getId(), t));
myCode2Concept = theCode2Concept;
myConceptMaps = theConceptMaps;
myConceptMaps.forEach(t -> myIdToConceptMaps.put(t.getId(), t));
myUploadProperties = theUploadProperties;
}
@ -192,7 +194,6 @@ public abstract class BaseLoincHandler implements IRecordHandler {
vs.setUrl(theValueSetUri);
vs.setId(theValueSetId);
vs.setVersion(version);
vs.setName(theValueSetName);
vs.setStatus(Enumerations.PublicationStatus.ACTIVE);
vs.setPublisher(REGENSTRIEF_INSTITUTE_INC);
vs.addContact()
@ -206,6 +207,11 @@ public abstract class BaseLoincHandler implements IRecordHandler {
} else {
vs = myIdToValueSet.get(theValueSetId);
}
if (isBlank(vs.getName()) && isNotBlank(theValueSetName)) {
vs.setName(theValueSetName);
}
return vs;
}

View File

@ -0,0 +1,62 @@
package ca.uhn.fhir.jpa.term.loinc;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2018 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.term.IRecordHandler;
import org.apache.commons.csv.CSVRecord;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static org.apache.commons.lang3.StringUtils.trim;
public class LoincGroupFileHandler extends BaseLoincHandler implements IRecordHandler {
public static final String VS_URI_PREFIX = "http://loinc.org/fhir/loinc-group-";
public LoincGroupFileHandler(Map<String, TermConcept> theCode2concept, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, Properties theUploadProperties) {
super(theCode2concept, theValueSets, theConceptMaps, theUploadProperties);
}
@Override
public void accept(CSVRecord theRecord) {
//"ParentGroupId","GroupId","Group","Archetype","Status","VersionFirstReleased"
String parentGroupId = trim(theRecord.get("ParentGroupId"));
String groupId = trim(theRecord.get("GroupId"));
String groupName = trim(theRecord.get("Group"));
ValueSet parentValueSet = getValueSet(parentGroupId, VS_URI_PREFIX + parentGroupId, null, null);
parentValueSet
.getCompose()
.getIncludeFirstRep()
.addValueSet(VS_URI_PREFIX + groupId);
// Create group to set its name (terms are added in a different
// handler)
getValueSet(groupId, VS_URI_PREFIX + groupId, groupName, null);
}
}

View File

@ -0,0 +1,53 @@
package ca.uhn.fhir.jpa.term.loinc;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2018 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc;
import ca.uhn.fhir.jpa.term.IRecordHandler;
import org.apache.commons.csv.CSVRecord;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static org.apache.commons.lang3.StringUtils.trim;
public class LoincGroupTermsFileHandler extends BaseLoincHandler implements IRecordHandler {
public LoincGroupTermsFileHandler(Map<String, TermConcept> theCode2concept, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, Properties theUploadProperties) {
super(theCode2concept, theValueSets, theConceptMaps, theUploadProperties);
}
@Override
public void accept(CSVRecord theRecord) {
//"Category","GroupId","Archetype","LoincNumber","LongCommonName"
String groupId = trim(theRecord.get("GroupId"));
String loincNumber = trim(theRecord.get("LoincNumber"));
ValueSet valueSet = getValueSet(groupId, LoincGroupFileHandler.VS_URI_PREFIX + groupId, null, null);
addCodeAsIncludeToValueSet(valueSet, IHapiTerminologyLoaderSvc.LOINC_URI, loincNumber, null);
}
}

View File

@ -0,0 +1,51 @@
package ca.uhn.fhir.jpa.term.loinc;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2018 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.entity.TermConcept;
import ca.uhn.fhir.jpa.term.IRecordHandler;
import org.apache.commons.csv.CSVRecord;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ValueSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static org.apache.commons.lang3.StringUtils.trim;
public class LoincParentGroupFileHandler extends BaseLoincHandler implements IRecordHandler {
public LoincParentGroupFileHandler(Map<String, TermConcept> theCode2concept, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps, Properties theUploadProperties) {
super(theCode2concept, theValueSets, theConceptMaps, theUploadProperties);
}
@Override
public void accept(CSVRecord theRecord) {
// "ParentGroupId","ParentGroup","Status"
String parentGroupId = trim(theRecord.get("ParentGroupId"));
String parentGroupName = trim(theRecord.get("ParentGroup"));
getValueSet(parentGroupId, LoincGroupFileHandler.VS_URI_PREFIX + parentGroupId, parentGroupName, null);
}
}

View File

@ -42,6 +42,9 @@ public class TerminologyUploaderProviderDstu3Test extends BaseResourceProviderDs
addFile(zos, TerminologyLoaderSvcImpl.LOINC_HIERARCHY_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_LINK_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_GROUP_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_GROUP_TERMS_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_PARENT_GROUP_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_PART_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_PART_LINK_FILE);
addFile(zos, TerminologyLoaderSvcImpl.LOINC_PART_RELATED_CODE_MAPPING_FILE);

View File

@ -67,7 +67,6 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
public void testExpandWithPropertyCoding() throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesToZip(files);
TerminologyLoaderSvcLoincTest.addLoincOptionalFilesToZip(files);
myLoader.loadLoinc(files.getFiles(), mySrd);
// Search by code
@ -121,7 +120,6 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
public void testExpandWithPropertyString() throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesToZip(files);
TerminologyLoaderSvcLoincTest.addLoincOptionalFilesToZip(files);
myLoader.loadLoinc(files.getFiles(), mySrd);
ValueSet input = new ValueSet();
@ -144,7 +142,6 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
public void testLookupWithProperties() throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesToZip(files);
TerminologyLoaderSvcLoincTest.addLoincOptionalFilesToZip(files);
myLoader.loadLoinc(files.getFiles(), mySrd);
IFhirResourceDaoCodeSystem.LookupCodeResult result = myCodeSystemDao.lookupCode(new StringType("10013-1"), new StringType(IHapiTerminologyLoaderSvc.LOINC_URI), null, mySrd);
@ -176,7 +173,6 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test {
public void testLookupWithPropertiesExplicit() throws Exception {
ZipCollectionBuilder files = new ZipCollectionBuilder();
TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesToZip(files);
TerminologyLoaderSvcLoincTest.addLoincOptionalFilesToZip(files);
myLoader.loadLoinc(files.getFiles(), mySrd);
IFhirResourceDaoCodeSystem.LookupCodeResult result = myCodeSystemDao.lookupCode(new StringType("10013-1"), new StringType(IHapiTerminologyLoaderSvc.LOINC_URI), null, mySrd);

View File

@ -93,7 +93,6 @@ public class TerminologyLoaderSvcLoincTest {
@Test
public void testLoadLoinc() throws Exception {
addLoincMandatoryFilesToZip(myFiles);
addLoincOptionalFilesToZip(myFiles);
// Actually do the load
mySvc.loadLoinc(myFiles.getFiles(), details);
@ -287,7 +286,7 @@ public class TerminologyLoaderSvcLoincTest {
// IEEE Medical Device Codes
conceptMap = conceptMaps.get(LoincIeeeMedicalDeviceCodeHandler.LOINC_IEEE_CM_ID);
ourLog.info(FhirContext.forR4().newXmlParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
ourLog.debug(FhirContext.forR4().newXmlParser().setPrettyPrint(true).encodeResourceToString(conceptMap));
assertEquals(LoincIeeeMedicalDeviceCodeHandler.LOINC_IEEE_CM_NAME, conceptMap.getName());
assertEquals(LoincIeeeMedicalDeviceCodeHandler.LOINC_IEEE_CM_URI, conceptMap.getUrl());
assertEquals(1, conceptMap.getGroup().size());
@ -309,6 +308,25 @@ public class TerminologyLoaderSvcLoincTest {
assertEquals(9, vs.getCompose().getInclude().get(0).getConcept().size());
assertEquals("11525-3", vs.getCompose().getInclude().get(0).getConcept().get(0).getCode());
assertEquals("US Pelvis Fetus for pregnancy", vs.getCompose().getInclude().get(0).getConcept().get(0).getDisplay());
// Group - Parent
vs = valueSets.get("LG100-4");
ourLog.info(FhirContext.forR4().newXmlParser().setPrettyPrint(true).encodeResourceToString(vs));
assertEquals("Chem_DrugTox_Chal_Sero_Allergy<SAME:Comp|Prop|Tm|Syst (except intravascular and urine)><ANYBldSerPlas,ANYUrineUrineSed><ROLLUP:Method>", vs.getName());
assertEquals("http://loinc.org/fhir/loinc-group-LG100-4", vs.getUrl());
assertEquals(1, vs.getCompose().getInclude().size());
assertEquals(1, vs.getCompose().getInclude().get(0).getValueSet().size());
assertEquals("http://loinc.org/fhir/loinc-group-LG1695-8", vs.getCompose().getInclude().get(0).getValueSet().get(0).getValueAsString());
// Group - Child
vs = valueSets.get("LG1695-8");
ourLog.info(FhirContext.forR4().newXmlParser().setPrettyPrint(true).encodeResourceToString(vs));
assertEquals("1,4-Dichlorobenzene|MCnc|Pt|ANYBldSerPl", vs.getName());
assertEquals("http://loinc.org/fhir/loinc-group-LG1695-8", vs.getUrl());
assertEquals(1, vs.getCompose().getInclude().size());
assertEquals(2, vs.getCompose().getInclude().get(0).getConcept().size());
assertEquals("17424-3", vs.getCompose().getInclude().get(0).getConcept().get(0).getCode());
assertEquals("13006-2", vs.getCompose().getInclude().get(0).getConcept().get(1).getCode());
}
@Test
@ -336,7 +354,8 @@ public class TerminologyLoaderSvcLoincTest {
@Test
public void testLoadLoincMissingMandatoryFiles() throws IOException {
addLoincOptionalFilesToZip(myFiles);
myFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_UPLOAD_PROPERTIES_FILE);
myFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_GROUP_FILE);
// Actually do the load
try {
@ -349,13 +368,13 @@ public class TerminologyLoaderSvcLoincTest {
public static void addLoincMandatoryFilesToZip(ZipCollectionBuilder theFiles) throws IOException {
theFiles.addFileZip("/loinc/", "Loinc.csv", TerminologyLoaderSvcImpl.LOINC_FILE);
theFiles.addFileZip("/loinc/", "MultiAxialHierarchy.csv", TerminologyLoaderSvcImpl.LOINC_HIERARCHY_FILE);
}
public static void addLoincOptionalFilesToZip(ZipCollectionBuilder theFiles) throws IOException {
theFiles.addFileZip("/loinc/", "loincupload.properties");
theFiles.addFileZip("/loinc/", "AnswerList.csv", TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_UPLOAD_PROPERTIES_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_GROUP_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_GROUP_TERMS_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_PARENT_GROUP_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_FILE, TerminologyLoaderSvcImpl.LOINC_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_HIERARCHY_FILE, TerminologyLoaderSvcImpl.LOINC_HIERARCHY_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_FILE, TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_LINK_FILE, TerminologyLoaderSvcImpl.LOINC_ANSWERLIST_LINK_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_PART_FILE, TerminologyLoaderSvcImpl.LOINC_PART_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_PART_LINK_FILE, TerminologyLoaderSvcImpl.LOINC_PART_LINK_FILE);
@ -365,13 +384,8 @@ public class TerminologyLoaderSvcLoincTest {
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_IMAGING_DOCUMENT_CODES_FILE);
/*
* Top 2000 files have versions in the filename so don't use the
* constant.. that way this is a better test
*/
theFiles.addFilePlain("/loinc/", TerminologyLoaderSvcImpl.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE);
theFiles.addFilePlain("/loinc/", TerminologyLoaderSvcImpl.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE);
theFiles.addFileZip("/loinc/", TerminologyLoaderSvcImpl.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE);
}
@AfterClass

View File

@ -0,0 +1,2 @@
"ParentGroupId","GroupId","Group","Archetype","Status","VersionFirstReleased"
"LG100-4","LG1695-8","1,4-Dichlorobenzene|MCnc|Pt|ANYBldSerPl","","Active",""
1 ParentGroupId GroupId Group Archetype Status VersionFirstReleased
2 LG100-4 LG1695-8 1,4-Dichlorobenzene|MCnc|Pt|ANYBldSerPl Active

View File

@ -0,0 +1,3 @@
"Category","GroupId","Archetype","LoincNumber","LongCommonName"
"Flowsheet","LG1695-8","","17424-3","1,4-Dichlorobenzene [Mass/volume] in Blood"
"Flowsheet","LG1695-8","","13006-2","1,4-Dichlorobenzene [Mass/volume] in Serum or Plasma"
1 Category GroupId Archetype LoincNumber LongCommonName
2 Flowsheet LG1695-8 17424-3 1,4-Dichlorobenzene [Mass/volume] in Blood
3 Flowsheet LG1695-8 13006-2 1,4-Dichlorobenzene [Mass/volume] in Serum or Plasma

View File

@ -0,0 +1,2 @@
"ParentGroupId","ParentGroup","Status"
"LG100-4","Chem_DrugTox_Chal_Sero_Allergy<SAME:Comp|Prop|Tm|Syst (except intravascular and urine)><ANYBldSerPlas,ANYUrineUrineSed><ROLLUP:Method>","ACTIVE"
1 ParentGroupId ParentGroup Status
2 LG100-4 Chem_DrugTox_Chal_Sero_Allergy<SAME:Comp|Prop|Tm|Syst (except intravascular and urine)><ANYBldSerPlas,ANYUrineUrineSed><ROLLUP:Method> ACTIVE