diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/LoadedFileDescriptors.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/LoadedFileDescriptors.java index 11ca46d9624..4d3ed5bb5c4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/LoadedFileDescriptors.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/LoadedFileDescriptors.java @@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.term; */ import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; +import ca.uhn.fhir.jpa.util.LogicUtil; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; import org.apache.commons.io.FileUtils; @@ -31,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -140,6 +142,24 @@ public class LoadedFileDescriptors implements Closeable { } } + void verifyPartLinkFilesExist(List theMultiPartLinkFiles, String theSinglePartLinkFile) { + List notFoundMulti = notFound(theMultiPartLinkFiles); + List notFoundSingle = notFound(Arrays.asList(theSinglePartLinkFile)); + // Expect all of the files in theMultiPartLinkFiles to be found and theSinglePartLinkFile to not be found, + // or none of the files in theMultiPartLinkFiles to be found and the SinglePartLinkFile to be found. + boolean multiPartFilesFound = notFoundMulti.isEmpty(); + boolean singlePartFilesFound = notFoundSingle.isEmpty(); + if (!LogicUtil.multiXor(multiPartFilesFound, singlePartFilesFound)) { + String msg; + if (!multiPartFilesFound && !singlePartFilesFound) { + msg = "Could not find any of the PartLink files: " + notFoundMulti + " or " + notFoundSingle; + } else { + msg = "Found both the single PartLink file, " + theSinglePartLinkFile + ", and the split PartLink files: " + theMultiPartLinkFiles; + } + throw new UnprocessableEntityException(msg); + } + } + private static class NonClosableBOMInputStream extends BOMInputStream { NonClosableBOMInputStream(InputStream theWrap) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java index f1e53b28d53..25a12a02b9f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java @@ -114,8 +114,6 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { uploadProperties.getProperty(LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE.getCode(), LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_IMAGING_DOCUMENT_CODES_FILE.getCode(), LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_PART_FILE.getCode(), LOINC_PART_FILE_DEFAULT.getCode()), - uploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), - uploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_PART_RELATED_CODE_MAPPING_FILE.getCode(), LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_RSNA_PLAYBOOK_FILE.getCode(), LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE.getCode(), LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode()), @@ -124,6 +122,12 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { ); descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments); + List splitPartLinkFilenameFragments = Arrays.asList( + uploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), + uploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()) + ); + descriptors.verifyPartLinkFilesExist(splitPartLinkFilenameFragments, uploadProperties.getProperty(LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode())); + List optionalFilenameFragments = Arrays.asList( uploadProperties.getProperty(LOINC_GROUP_FILE.getCode(), LOINC_GROUP_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_GROUP_TERMS_FILE.getCode(), LOINC_GROUP_TERMS_FILE_DEFAULT.getCode()), @@ -475,8 +479,9 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { // Part link handler = new LoincPartLinkHandler(codeSystemVersion, code2concept, propertyNamesToTypes); - iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); - iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + iterateOverZipFileOptional(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + iterateOverZipFileOptional(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + iterateOverZipFileOptional(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); IOUtils.closeQuietly(theDescriptors); @@ -598,6 +603,14 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { } public static void iterateOverZipFile(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode, boolean theIsPartialFilename) { + iterateOverZipFile(theDescriptors, theFileNamePart, theHandler, theDelimiter, theQuoteMode, theIsPartialFilename, true); + } + + public static void iterateOverZipFileOptional(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode, boolean theIsPartialFilename) { + iterateOverZipFile(theDescriptors, theFileNamePart, theHandler, theDelimiter, theQuoteMode, theIsPartialFilename, false); + } + + private static void iterateOverZipFile(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode, boolean theIsPartialFilename, boolean theRequireMatch) { boolean foundMatch = false; for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) { @@ -644,7 +657,7 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { } - if (!foundMatch) { + if (!foundMatch && theRequireMatch) { throw new InvalidRequestException("Did not find file matching " + theFileNamePart); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java index af8c414c79a..1c0b5330d3c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java @@ -69,6 +69,8 @@ public enum LoincUploadPropertiesEnum { LOINC_PART_FILE_DEFAULT("AccessoryFiles/PartFile/Part.csv"), // Part link + LOINC_PART_LINK_FILE("loinc.part.link.file"), + LOINC_PART_LINK_FILE_DEFAULT("AccessoryFiles/PartFile/LoincPartLink.csv"), LOINC_PART_LINK_FILE_PRIMARY("loinc.part.link.primary.file"), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT("AccessoryFiles/PartFile/LoincPartLink_Primary.csv"), LOINC_PART_LINK_FILE_SUPPLEMENTARY("loinc.part.link.supplementary.file"), diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java index d143f838bda..feb806040a8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java @@ -68,9 +68,54 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { } @Test - public void testLoadLoinc() throws Exception { + public void testLoadLoincWithSplitPartLink() throws Exception { addLoincMandatoryFilesToZip(myFiles); + verifyLoadLoinc(); + } + @Test + public void testLoadLoincWithSinglePartLink() throws Exception { + addLoincMandatoryFilesAndSinglePartLinkToZip(myFiles); + verifyLoadLoinc(); + } + + @Test + public void testLoadLoincInvalidPartLinkFiles() throws IOException { + + // Missing all PartLinkFiles + addBaseLoincMandatoryFilesToZip(myFiles); + myFiles.addFileZip("/loinc/", LOINC_UPLOAD_PROPERTIES_FILE.getCode()); + + try { + mySvc.loadLoinc(myFiles.getFiles(), mySrd); + fail(); + } catch (UnprocessableEntityException e) { + assertEquals("Could not find any of the PartLink files: [AccessoryFiles/PartFile/LoincPartLink_Primary.csv, AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv] or [AccessoryFiles/PartFile/LoincPartLink.csv]", e.getMessage()); + } + + // Missing LoincPartLink_Supplementary + myFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); + try { + mySvc.loadLoinc(myFiles.getFiles(), mySrd); + fail(); + } catch (UnprocessableEntityException e) { + assertEquals("Could not find any of the PartLink files: [AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv] or [AccessoryFiles/PartFile/LoincPartLink.csv]", e.getMessage()); + } + + // Both Split and Single PartLink files + myFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); + myFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_DEFAULT.getCode()); + try { + mySvc.loadLoinc(myFiles.getFiles(), mySrd); + fail(); + } catch (UnprocessableEntityException e) { + assertEquals("Found both the single PartLink file, AccessoryFiles/PartFile/LoincPartLink.csv, and the split PartLink files: [AccessoryFiles/PartFile/LoincPartLink_Primary.csv, AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv]", e.getMessage()); + } + + } + + + private void verifyLoadLoinc() throws Exception { // Actually do the load mySvc.loadLoinc(myFiles.getFiles(), mySrd); @@ -413,12 +458,27 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { } } + public static void addLoincMandatoryFilesAndSinglePartLinkToZip(ZipCollectionBuilder theFiles) throws IOException { + addBaseLoincMandatoryFilesToZip(theFiles); + theFiles.addFileZip("/loinc/", "loincupload_singlepartlink.properties"); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_DEFAULT.getCode()); + } + public static void addLoincMandatoryFilesToZip(ZipCollectionBuilder theFiles) throws IOException { - addLoincMandatoryFilesWithPropertiesFileToZip(theFiles, LOINC_UPLOAD_PROPERTIES_FILE.getCode()); + addBaseLoincMandatoryFilesToZip(theFiles); + theFiles.addFileZip("/loinc/", LOINC_UPLOAD_PROPERTIES_FILE.getCode()); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); } public static void addLoincMandatoryFilesWithPropertiesFileToZip(ZipCollectionBuilder theFiles, String thePropertiesFile) throws IOException { theFiles.addFileZip("/loinc/", thePropertiesFile); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); + addBaseLoincMandatoryFilesToZip(theFiles); + } + + private static void addBaseLoincMandatoryFilesToZip(ZipCollectionBuilder theFiles) throws IOException{ theFiles.addFileZip("/loinc/", LOINC_GROUP_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_GROUP_TERMS_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_PARENT_GROUP_FILE_DEFAULT.getCode()); @@ -430,8 +490,6 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { theFiles.addFileZip("/loinc/", LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_PART_FILE_DEFAULT.getCode()); - theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); - theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()); diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv new file mode 100644 index 00000000000..36c84f3fe95 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv @@ -0,0 +1,10 @@ +"LoincNumber","LongCommonName","PartNumber","PartName","PartCodeSystem","PartTypeName","LinkTypeName","Property" +"10013-1","R' wave amplitude in lead I","LP31101-6","R' wave amplitude.lead I","http://loinc.org","COMPONENT","Primary","http://loinc.org/property/COMPONENT" +"10013-1","R' wave amplitude in lead I","LP6802-5","Elpot","http://loinc.org","PROPERTY","Primary","http://loinc.org/property/PROPERTY" +"10013-1","R' wave amplitude in lead I","LP6960-1","Pt","http://loinc.org","TIME","Primary","http://loinc.org/property/TIME_ASPCT" +"10013-1","R' wave amplitude in lead I","LP7289-4","Heart","http://loinc.org","SYSTEM","Primary","http://loinc.org/property/SYSTEM" +"10013-1","R' wave amplitude in lead I","LP7753-9","Qn","http://loinc.org","SCALE","Primary","http://loinc.org/property/SCALE_TYP" +"10013-1","R' wave amplitude in lead I","LP6244-0","EKG","http://loinc.org","METHOD","Primary","http://loinc.org/property/METHOD_TYP" +"10013-1","R' wave amplitude in lead I","LP31101-6","R' wave amplitude.lead I","http://loinc.org","COMPONENT","DetailedModel","http://loinc.org/property/analyte" +"10013-1","R' wave amplitude in lead I","LP6802-5","Elpot","http://loinc.org","PROPERTY","DetailedModel","http://loinc.org/property/PROPERTY" +"10013-1","R' wave amplitude in lead I","LP6960-1","Pt","http://loinc.org","TIME","DetailedModel","http://loinc.org/property/time-core" diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload_singlepartlink.properties b/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload_singlepartlink.properties new file mode 100644 index 00000000000..3164f533888 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload_singlepartlink.properties @@ -0,0 +1,82 @@ +################# +### MANDATORY ### +################# + +# Answer lists (ValueSets of potential answers/values for LOINC "questions") +## File must be present +loinc.answerlist.file=AccessoryFiles/AnswerFile/AnswerList.csv +# Answer list links (connects LOINC observation codes to answer list codes) +## File must be present +loinc.answerlist.link.file=AccessoryFiles/AnswerFile/LoincAnswerListLink.csv + +# Document ontology +## File must be present +loinc.document.ontology.file=AccessoryFiles/DocumentOntology/DocumentOntology.csv + +# LOINC codes +## File must be present +loinc.file=LoincTable/Loinc.csv + +# LOINC hierarchy +## File must be present +loinc.hierarchy.file=AccessoryFiles/MultiAxialHierarchy/MultiAxialHierarchy.csv + +# IEEE medical device codes +## File must be present +loinc.ieee.medical.device.code.mapping.table.file=AccessoryFiles/LoincIeeeMedicalDeviceCodeMappingTable/LoincIeeeMedicalDeviceCodeMappingTable.csv + +# Imaging document codes +## File must be present +loinc.imaging.document.codes.file=AccessoryFiles/ImagingDocuments/ImagingDocumentCodes.csv + +# Part +## File must be present +loinc.part.file=AccessoryFiles/PartFile/Part.csv + +# Part link +## File must be present +loinc.part.link.file=AccessoryFiles/PartFile/LoincPartLink.csv + +# Part related code mapping +## File must be present +loinc.part.related.code.mapping.file=AccessoryFiles/PartFile/PartRelatedCodeMapping.csv + +# RSNA playbook +## File must be present +loinc.rsna.playbook.file=AccessoryFiles/LoincRsnaRadiologyPlaybook/LoincRsnaRadiologyPlaybook.csv + +# Top 2000 codes - SI +## File must be present +loinc.top2000.common.lab.results.si.file=AccessoryFiles/Top2000Results/SI/Top2000CommonLabResultsSi.csv +# Top 2000 codes - US +## File must be present +loinc.top2000.common.lab.results.us.file=AccessoryFiles/Top2000Results/US/Top2000CommonLabResultsUs.csv + +# Universal lab order ValueSet +## File must be present +loinc.universal.lab.order.valueset.file=AccessoryFiles/LoincUniversalLabOrdersValueSet/LoincUniversalLabOrdersValueSet.csv + +################ +### OPTIONAL ### +################ + +# This is the version identifier for the answer list file +## Key may be omitted +loinc.answerlist.version=Beta.1 + +# This is the version identifier for uploaded ConceptMap resources +## Key may be omitted +loinc.conceptmap.version=Beta.1 + +# Group +## Default value if key not provided: AccessoryFiles/GroupFile/Group.csv +## File may be omitted +loinc.group.file=AccessoryFiles/GroupFile/Group.csv +# Group terms +## Default value if key not provided: AccessoryFiles/GroupFile/GroupLoincTerms.csv +## File may be omitted +loinc.group.terms.file=AccessoryFiles/GroupFile/GroupLoincTerms.csv +# Parent group +## Default value if key not provided: AccessoryFiles/GroupFile/ParentGroup.csv +## File may be omitted +loinc.parent.group.file=AccessoryFiles/GroupFile/ParentGroup.csv