diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java index 3d02e64e2f0..fbca3fa82f9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java @@ -39,6 +39,7 @@ import org.hl7.fhir.dstu3.model.CodeableConcept; import org.hl7.fhir.dstu3.model.Coding; import org.hl7.fhir.dstu3.model.IdType; import org.hl7.fhir.dstu3.model.ValueSet; +import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent; import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent; import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent; import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome; @@ -53,6 +54,7 @@ import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet; import ca.uhn.fhir.jpa.util.LogicUtil; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.util.ElementUtil; public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 implements IFhirResourceDaoValueSet { @@ -70,12 +72,15 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 return retVal; } - private ValueSet doExpand(ValueSet source, String theFilter) { + private ValueSet doExpand(ValueSet theSource, String theFilter) { + + validateIncludes("include", theSource.getCompose().getInclude()); + validateIncludes("exclude", theSource.getCompose().getExclude()); HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport); String filterLc = theFilter != null ? theFilter.toLowerCase() : null; - ValueSetExpansionOutcome outcome = workerContext.expand(source); + ValueSetExpansionOutcome outcome = workerContext.expand(theSource); ValueSetExpansionComponent expansion = outcome.getValueset().getExpansion(); if (isNotBlank(theFilter)) { for (Iterator containsIter = expansion.getContains().iterator(); containsIter.hasNext();) { @@ -91,6 +96,14 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 return retVal; } + private void validateIncludes(String name, List listToValidate) { + for (ConceptSetComponent nextExclude : listToValidate) { + if (isBlank(nextExclude.getSystem()) && !ElementUtil.isEmpty(nextExclude.getConcept(), nextExclude.getFilter())) { + throw new InvalidRequestException("ValueSet contains " + name + " criteria with no system defined"); + } + } + } + @Override public ValueSet expandByIdentifier(String theUri, String theFilter) { if (isBlank(theUri)) { @@ -120,55 +133,6 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3 ValueSet retVal = doExpand(source, theFilter); return retVal; -// ValueSet retVal = new ValueSet(); -// retVal.setDate(new Date()); -// -// /* -// * Add composed concepts -// */ -// -// for (ConceptSetComponent nextInclude : source.getCompose().getInclude()) { -// if (nextInclude.getConcept().isEmpty()) { -// -// } else { -// for (ConceptReferenceComponent next : nextInclude.getConcept()) { -// if (isBlank(theFilter)) { -// addCompose(retVal, nextInclude.getSystem(), next.getCode(), next.getDisplay()); -// } else { -// String filter = theFilter.toLowerCase(); -// if (next.getDisplay().toLowerCase().contains(filter) || next.getCode().toLowerCase().contains(filter)) { -// addCompose(retVal, nextInclude.getSystem(), next.getCode(), next.getDisplay()); -// } -// } -// } -// } -// } -// -// return retVal; - } - - private void addCompose(String theFilter, ValueSet theValueSetToPopulate, ValueSet theSourceValueSet, ConceptDefinitionComponent theConcept, String theSystem) { - if (isBlank(theFilter)) { - addCompose(theValueSetToPopulate, theSystem, theConcept.getCode(), theConcept.getDisplay()); - } else { - String filter = theFilter.toLowerCase(); - if (theConcept.getDisplay().toLowerCase().contains(filter) || theConcept.getCode().toLowerCase().contains(filter)) { - addCompose(theValueSetToPopulate, theSystem, theConcept.getCode(), theConcept.getDisplay()); - } - } - for (ConceptDefinitionComponent nextChild : theConcept.getConcept()) { - addCompose(theFilter, theValueSetToPopulate, theSourceValueSet, nextChild, theSystem); - } - } - - private void addCompose(ValueSet retVal, String theSystem, String theCode, String theDisplay) { - if (isBlank(theCode)) { - return; - } - ValueSetExpansionContainsComponent contains = retVal.getExpansion().addContains(); - contains.setSystem(theSystem); - contains.setCode(theCode); - contains.setDisplay(theDisplay); } @Override diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3.java index 5eedc2f9cc2..2aa4eff6255 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3.java @@ -68,6 +68,8 @@ public class TerminologyUploaderProviderDstu3 extends BaseJpaProvider { UploadStatistics stats; if (IHapiTerminologyLoaderSvc.SCT_URL.equals(url)) { stats = myTerminologyLoaderSvc.loadSnomedCt(data, theRequestDetails); + } else if (IHapiTerminologyLoaderSvc.LOINC_URL.equals(url)) { + stats = myTerminologyLoaderSvc.loadLoinc(data, theRequestDetails); } else { throw new InvalidRequestException("Unknown URL: " + url); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologyLoaderSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologyLoaderSvc.java index 4b4d02f6d9e..2b254917330 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologyLoaderSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologyLoaderSvc.java @@ -4,21 +4,24 @@ import ca.uhn.fhir.rest.method.RequestDetails; public interface IHapiTerminologyLoaderSvc { + String LOINC_URL = "http://loinc.org"; String SCT_URL = "http://snomed.info/sct"; + UploadStatistics loadLoinc(byte[] theZipBytes, RequestDetails theRequestDetails); + UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails); public static class UploadStatistics { - private int myConceptCount; + private final int myConceptCount; + + public UploadStatistics(int theConceptCount) { + myConceptCount = theConceptCount; + } public int getConceptCount() { return myConceptCount; } - public UploadStatistics setConceptCount(int theConceptCount) { - myConceptCount = theConceptCount; - return this; - } } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvc.java index 31ac8a0692e..7af760e7777 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvc.java @@ -1,5 +1,7 @@ package ca.uhn.fhir.jpa.term; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + /* * #%L * HAPI FHIR JPA Server @@ -45,9 +47,11 @@ import java.util.zip.ZipInputStream; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; import org.springframework.beans.factory.annotation.Autowired; import com.google.common.annotations.VisibleForTesting; @@ -58,17 +62,26 @@ import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; import ca.uhn.fhir.rest.method.RequestDetails; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.util.CoverageIgnore; public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { + public static final String LOINC_FILE = "loinc.csv"; + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvc.class); - static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full"; - static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full"; - - static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full"; + public static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_"; + public static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full-en"; + public static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full"; - @Autowired private IHapiTerminologySvc myTermSvc; + + private void cleanUpTemporaryFiles(Map filenameToFile) { + ourLog.info("Finished terminology file import, cleaning up temporary files"); + for (File nextFile : filenameToFile.values()) { + nextFile.delete(); + } + } private void dropCircularRefs(TermConcept theConcept, LinkedHashSet theChain, Map theCode2concept) { @@ -101,56 +114,7 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { } - private TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map id2concept, String id) { - TermConcept concept = id2concept.get(id); - if (concept == null) { - concept = new TermConcept(); - id2concept.put(id, concept); - concept.setCodeSystem(codeSystemVersion); - } - return concept; - } - - private void iterateOverZipFile(Map theFilenameToFile, String fileNamePart, IRecordHandler handler) { - for (Entry nextEntry : theFilenameToFile.entrySet()) { - - if (nextEntry.getKey().contains(fileNamePart)) { - ourLog.info("Processing file {}", nextEntry.getKey()); - - Reader reader = null; - CSVParser parsed = null; - try { - reader = new BufferedReader(new FileReader(nextEntry.getValue())); - parsed = new CSVParser(reader, CSVFormat.newFormat('\t').withFirstRecordAsHeader()); - Iterator iter = parsed.iterator(); - ourLog.debug("Header map: {}", parsed.getHeaderMap()); - - int count = 0; - int logIncrement = 100000; - int nextLoggedCount = logIncrement; - while (iter.hasNext()) { - CSVRecord nextRecord = iter.next(); - handler.accept(nextRecord); - count++; - if (count >= nextLoggedCount) { - ourLog.info(" * Processed {} records in {}", count, fileNamePart); - nextLoggedCount += logIncrement; - } - } - } catch (IOException e) { - throw new InternalErrorException(e); - } finally { - IOUtils.closeQuietly(parsed); - IOUtils.closeQuietly(reader); - } - } - } - } - - @Override - public UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails) { - List allFilenames = Arrays.asList(SCT_FILE_DESCRIPTION, SCT_FILE_RELATIONSHIP, SCT_FILE_CONCEPT); - + private Map extractFiles(byte[] theZipBytes, List theExpectedFilenameFragments) { Map filenameToFile = new HashMap(); ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new ByteArrayInputStream(theZipBytes))); try { @@ -158,7 +122,7 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { ZippedFileInputStream inputStream = new ZippedFileInputStream(zis); boolean want = false; - for (String next : allFilenames) { + for (String next : theExpectedFilenameFragments) { if (nextEntry.getName().contains(next)) { want = true; } @@ -188,18 +152,112 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { IOUtils.closeQuietly(zis); } + if (filenameToFile.size() != theExpectedFilenameFragments.size()) { + throw new InvalidRequestException("Invalid input zip file, expected zip to contain the following name fragments: " + theExpectedFilenameFragments + " but found: " + filenameToFile.keySet()); + } + return filenameToFile; + } + + private TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map id2concept, String id) { + TermConcept concept = id2concept.get(id); + if (concept == null) { + concept = new TermConcept(); + id2concept.put(id, concept); + concept.setCodeSystem(codeSystemVersion); + } + return concept; + } + + private void iterateOverZipFile(Map theFilenameToFile, String fileNamePart, IRecordHandler handler, char theDelimiter, QuoteMode theQuoteMode) { + boolean found = false; + for (Entry nextEntry : theFilenameToFile.entrySet()) { + + if (nextEntry.getKey().contains(fileNamePart)) { + ourLog.info("Processing file {}", nextEntry.getKey()); + found = true; + + Reader reader = null; + CSVParser parsed = null; + try { + reader = new BufferedReader(new FileReader(nextEntry.getValue())); + CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader(); + if (theQuoteMode != null) { + format = format.withQuote('"').withQuoteMode(theQuoteMode); + } + parsed = new CSVParser(reader, format); + Iterator iter = parsed.iterator(); + ourLog.debug("Header map: {}", parsed.getHeaderMap()); + + int count = 0; + int logIncrement = 100000; + int nextLoggedCount = logIncrement; + while (iter.hasNext()) { + CSVRecord nextRecord = iter.next(); + handler.accept(nextRecord); + count++; + if (count >= nextLoggedCount) { + ourLog.info(" * Processed {} records in {}", count, fileNamePart); + nextLoggedCount += logIncrement; + } + } + } catch (IOException e) { + throw new InternalErrorException(e); + } finally { + IOUtils.closeQuietly(parsed); + IOUtils.closeQuietly(reader); + } + } + } + + // This should always be true, but just in case we've introduced a bug... + Validate.isTrue(found); + } + + @Override + public UploadStatistics loadLoinc(byte[] theZipBytes, RequestDetails theRequestDetails) { + List expectedFilenameFragments = Arrays.asList(LOINC_FILE); + + Map filenameToFile = extractFiles(theZipBytes, expectedFilenameFragments); + + ourLog.info("Beginning LOINC processing"); + + try { + return processLoincFiles(filenameToFile, theRequestDetails); + } finally { + cleanUpTemporaryFiles(filenameToFile); + } + } + + @Override + public UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails) { + List expectedFilenameFragments = Arrays.asList(SCT_FILE_DESCRIPTION, SCT_FILE_RELATIONSHIP, SCT_FILE_CONCEPT); + + Map filenameToFile = extractFiles(theZipBytes, expectedFilenameFragments); + ourLog.info("Beginning SNOMED CT processing"); try { return processSnomedCtFiles(filenameToFile, theRequestDetails); } finally { - ourLog.info("Finished SNOMED CT file import, cleaning up temporary files"); - for (File nextFile : filenameToFile.values()) { - nextFile.delete(); - } + cleanUpTemporaryFiles(filenameToFile); } } + UploadStatistics processLoincFiles(Map filenameToFile, RequestDetails theRequestDetails) { + final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion(); + final Map code2concept = new HashMap(); + + IRecordHandler handler = new LoincHandler(codeSystemVersion, code2concept); + iterateOverZipFile(filenameToFile, LOINC_FILE, handler, ',', QuoteMode.NON_NUMERIC); + + ourLog.info("Have {} concepts", code2concept.size()); + + codeSystemVersion.getConcepts().addAll(code2concept.values()); + myTermSvc.storeNewCodeSystemVersion(SCT_URL, codeSystemVersion, theRequestDetails); + + return new UploadStatistics(code2concept.size()); + } + UploadStatistics processSnomedCtFiles(Map filenameToFile, RequestDetails theRequestDetails) { final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion(); final Map id2concept = new HashMap(); @@ -207,18 +265,18 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { final Set validConceptIds = new HashSet(); IRecordHandler handler = new SctHandlerConcept(validConceptIds); - iterateOverZipFile(filenameToFile, SCT_FILE_CONCEPT, handler); + iterateOverZipFile(filenameToFile, SCT_FILE_CONCEPT, handler,'\t', null); ourLog.info("Have {} valid concept IDs", validConceptIds.size()); handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion); - iterateOverZipFile(filenameToFile, SCT_FILE_DESCRIPTION, handler); + iterateOverZipFile(filenameToFile, SCT_FILE_DESCRIPTION, handler,'\t', null); ourLog.info("Got {} concepts, cloning map", code2concept.size()); final HashMap rootConcepts = new HashMap(code2concept); handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept); - iterateOverZipFile(filenameToFile, SCT_FILE_RELATIONSHIP, handler); + iterateOverZipFile(filenameToFile, SCT_FILE_RELATIONSHIP, handler,'\t', null); ourLog.info("Done loading SNOMED CT files - {} root codes, {} total codes", rootConcepts.size(), code2concept.size()); @@ -229,14 +287,15 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { codeSystemVersion.getConcepts().addAll(rootConcepts.values()); myTermSvc.storeNewCodeSystemVersion(SCT_URL, codeSystemVersion, theRequestDetails); - return new UploadStatistics().setConceptCount(code2concept.size()); + return new UploadStatistics(code2concept.size()); } - + @VisibleForTesting void setTermSvcForUnitTests(IHapiTerminologySvc theTermSvc) { myTermSvc = theTermSvc; } + @CoverageIgnore public static void main(String[] args) throws Exception { TerminologyLoaderSvc svc = new TerminologyLoaderSvc(); @@ -254,6 +313,33 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { void accept(CSVRecord theRecord); } + public class LoincHandler implements IRecordHandler { + + private final Map myCode2Concept; + private final TermCodeSystemVersion myCodeSystemVersion; + + public LoincHandler(TermCodeSystemVersion theCodeSystemVersion, Map theCode2concept) { + myCodeSystemVersion = theCodeSystemVersion; + myCode2Concept = theCode2concept; + } + + @Override + public void accept(CSVRecord theRecord) { + String code = theRecord.get("LOINC_NUM"); + String longCommonName = theRecord.get("LONG_COMMON_NAME"); + String shortName = theRecord.get("SHORTNAME"); + String consumerName = theRecord.get("CONSUMER_NAME"); + String display = firstNonBlank(longCommonName, shortName, consumerName); + + TermConcept concept = new TermConcept(myCodeSystemVersion, code); + concept.setDisplay(display); + + Validate.isTrue(!myCode2Concept.containsKey(code)); + myCode2Concept.put(code, concept); + } + + } + private final class SctHandlerConcept implements IRecordHandler { private Set myValidConceptIds; @@ -436,4 +522,15 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc { } } + public String firstNonBlank(String... theStrings) { + String retVal = ""; + for (String nextString : theStrings) { + if (isNotBlank(nextString)) { + retVal = nextString; + break; + } + } + return retVal; + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java index 47de80e07e9..cc0da5e0242 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java @@ -339,6 +339,51 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test { assertEquals(URL_MY_CODE_SYSTEM, result.getExpansion().getContains().get(idx).getSystem()); } + @Test + public void testExpandWithExcludeInExternalValueSet() { + createExternalCsAndLocalVs(); + + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + + ConceptSetComponent exclude = vs.getCompose().addExclude(); + exclude.setSystem(URL_MY_CODE_SYSTEM); + exclude.addConcept().setCode("childAA"); + exclude.addConcept().setCode("childAAA"); + + ValueSet result = myValueSetDao.expand(vs, null); + + String encoded = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result); + ourLog.info(encoded); + + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, containsInAnyOrder("ParentA", "ParentB", "childAB", "childAAB")); + + } + + @Test + public void testExpandWithInvalidExclude() { + createExternalCsAndLocalVs(); + + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + + /* + * No system set on exclude + */ + ConceptSetComponent exclude = vs.getCompose().addExclude(); + exclude.addConcept().setCode("childAA"); + exclude.addConcept().setCode("childAAA"); + try { + myValueSetDao.expand(vs, null); + fail(); + } catch (InvalidRequestException e) { + assertEquals("ValueSet contains exclude criteria with no system defined", e.getMessage()); + } + } + @Test public void testExpandWithSystemAndCodesAndFilterKeywordInLocalValueSet() { createLocalCsAndVs(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3Test.java similarity index 80% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyProviderDstu3Test.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3Test.java index 63d61feaa22..c2a9530d92b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3Test.java @@ -27,9 +27,9 @@ import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.TestUtil; -public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test { +public class TerminologyUploaderProviderDstu3Test extends BaseResourceProviderDstu3Test { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyProviderDstu3Test.class); + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProviderDstu3Test.class); @AfterClass public static void afterClassClearContext() { @@ -58,6 +58,26 @@ public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test assertThat(((IntegerType)respParam.getParameter().get(0).getValue()).getValue(), greaterThan(1)); } + @Test + public void testUploadLoinc() throws Exception { + byte[] packageBytes = createLoincZip(); + + //@formatter:off + Parameters respParam = ourClient + .operation() + .onServer() + .named("upload-external-code-system") + .withParameter(Parameters.class, "url", new UriType(IHapiTerminologyLoaderSvc.LOINC_URL)) + .andParameter("package", new Attachment().setData(packageBytes)) + .execute(); + //@formatter:on + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(respParam); + ourLog.info(resp); + + assertThat(((IntegerType)respParam.getParameter().get(0).getValue()).getValue(), greaterThan(1)); + } + @Test public void testUploadSctLocalFile() throws Exception { byte[] packageBytes = createSctZip(); @@ -154,4 +174,16 @@ public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test return packageBytes; } + private byte[] createLoincZip() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(bos); + + zos.putNextEntry(new ZipEntry("loinc.csv")); + zos.write(IOUtils.toByteArray(getClass().getResourceAsStream("/loinc/loinc.csv"))); + zos.close(); + + byte[] packageBytes = bos.toByteArray(); + return packageBytes; + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationTest.java deleted file mode 100644 index 4b6bfaab01c..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package ca.uhn.fhir.jpa.term; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import ca.uhn.fhir.jpa.dao.dstu3.BaseJpaDstu3Test; -import ca.uhn.fhir.util.TestUtil; - -public class TerminologyLoaderSvcIntegrationTest extends BaseJpaDstu3Test { - - private TerminologyLoaderSvc myLoader; - - @AfterClass - public static void afterClassClearContext() { - TestUtil.clearAllStaticFieldsForUnitTest(); - } - - @Before - public void beforeInitTest() { - myLoader = new TerminologyLoaderSvc(); - myLoader.setTermSvcForUnitTests(myTermSvc); - } - - @Test - @Ignore - public void testLoadAndStoreSnomedCt() { - Map files = new HashMap(); - files.put(TerminologyLoaderSvc.SCT_FILE_CONCEPT, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Concept_Full_INT_20160131.txt")); - files.put(TerminologyLoaderSvc.SCT_FILE_DESCRIPTION, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Description_Full-en_INT_20160131.txt")); - files.put(TerminologyLoaderSvc.SCT_FILE_RELATIONSHIP, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Relationship_Full_INT_20160131.txt")); - myLoader.processSnomedCtFiles(files, mySrd); - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcTest.java index 41f58b7fa25..9225e52d4f5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcTest.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.term; +import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import java.io.ByteArrayOutputStream; @@ -15,6 +16,7 @@ import org.junit.Ignore; import org.junit.Test; import ca.uhn.fhir.rest.method.RequestDetails; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.TestUtil; public class TerminologyLoaderSvcTest { @@ -34,18 +36,31 @@ public class TerminologyLoaderSvcTest { public static void afterClassClearContext() { TestUtil.clearAllStaticFieldsForUnitTest(); } - + + @Test + public void testLoadLoinc() throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(bos); + addEntry(zos,"/loinc/", "loinc.csv"); + zos.close(); + + ourLog.info("ZIP file has {} bytes", bos.toByteArray().length); + + RequestDetails details = mock(RequestDetails.class); + mySvc.loadLoinc(bos.toByteArray(), details); + } + @Test public void testLoadSnomedCt() throws Exception { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ZipOutputStream zos = new ZipOutputStream(bos); - addEntry(zos, "sct2_Concept_Full_INT_20160131.txt"); - addEntry(zos, "sct2_Concept_Full-en_INT_20160131.txt"); - addEntry(zos, "sct2_Description_Full-en_INT_20160131.txt"); - addEntry(zos, "sct2_Identifier_Full_INT_20160131.txt"); - addEntry(zos, "sct2_Relationship_Full_INT_20160131.txt"); - addEntry(zos, "sct2_StatedRelationship_Full_INT_20160131.txt"); - addEntry(zos, "sct2_TextDefinition_Full-en_INT_20160131.txt"); + addEntry(zos, "/sct/", "sct2_Concept_Full_INT_20160131.txt"); + addEntry(zos, "/sct/", "sct2_Concept_Full-en_INT_20160131.txt"); + addEntry(zos, "/sct/", "sct2_Description_Full-en_INT_20160131.txt"); + addEntry(zos, "/sct/", "sct2_Identifier_Full_INT_20160131.txt"); + addEntry(zos,"/sct/", "sct2_Relationship_Full_INT_20160131.txt"); + addEntry(zos,"/sct/", "sct2_StatedRelationship_Full_INT_20160131.txt"); + addEntry(zos, "/sct/", "sct2_TextDefinition_Full-en_INT_20160131.txt"); zos.close(); ourLog.info("ZIP file has {} bytes", bos.toByteArray().length); @@ -54,10 +69,28 @@ public class TerminologyLoaderSvcTest { mySvc.loadSnomedCt(bos.toByteArray(), details); } - private void addEntry(ZipOutputStream zos, String fileName) throws IOException { - ourLog.info("Adding {} to test zip", fileName); - zos.putNextEntry(new ZipEntry("SnomedCT_Release_INT_20160131_Full/Terminology/" + fileName)); - byte[] byteArray = IOUtils.toByteArray(getClass().getResourceAsStream("/sct/" + fileName)); + @Test + public void testLoadSnomedCtBadInput() throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(bos); + addEntry(zos, "/sct/", "sct2_StatedRelationship_Full_INT_20160131.txt"); + zos.close(); + + ourLog.info("ZIP file has {} bytes", bos.toByteArray().length); + + RequestDetails details = mock(RequestDetails.class); + try { + mySvc.loadSnomedCt(bos.toByteArray(), details); + fail(); + } catch (InvalidRequestException e) { + assertEquals("Invalid input zip file, expected zip to contain the following name fragments: [Terminology/sct2_Description_Full-en, Terminology/sct2_Relationship_Full, Terminology/sct2_Concept_Full_] but found: []", e.getMessage()); + } + } + + private void addEntry(ZipOutputStream zos, String theClasspathPrefix, String theFileName) throws IOException { + ourLog.info("Adding {} to test zip", theFileName); + zos.putNextEntry(new ZipEntry("SnomedCT_Release_INT_20160131_Full/Terminology/" + theFileName)); + byte[] byteArray = IOUtils.toByteArray(getClass().getResourceAsStream(theClasspathPrefix + theFileName)); Validate.notNull(byteArray); zos.write(byteArray); zos.closeEntry(); diff --git a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/terminologies/ValueSetExpanderSimple.java b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/terminologies/ValueSetExpanderSimple.java index 2f7fa2d6549..53e5f3b1bc5 100644 --- a/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/terminologies/ValueSetExpanderSimple.java +++ b/hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/terminologies/ValueSetExpanderSimple.java @@ -36,20 +36,22 @@ POSSIBILITY OF SUCH DAMAGE. import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.NotImplementedException; import org.hl7.fhir.dstu3.exceptions.TerminologyServiceException; +import org.hl7.fhir.dstu3.model.CodeSystem; +import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode; +import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent; import org.hl7.fhir.dstu3.model.DateTimeType; import org.hl7.fhir.dstu3.model.Factory; import org.hl7.fhir.dstu3.model.PrimitiveType; import org.hl7.fhir.dstu3.model.Type; import org.hl7.fhir.dstu3.model.UriType; import org.hl7.fhir.dstu3.model.ValueSet; -import org.hl7.fhir.dstu3.model.CodeSystem; -import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode; -import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent; import org.hl7.fhir.dstu3.model.ValueSet.ConceptReferenceComponent; import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent; import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent; @@ -59,221 +61,233 @@ import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent; import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent; import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionParameterComponent; import org.hl7.fhir.dstu3.utils.IWorkerContext; -import org.hl7.fhir.dstu3.utils.ToolingExtensions; import org.hl7.fhir.utilities.Utilities; public class ValueSetExpanderSimple implements ValueSetExpander { - private IWorkerContext context; - private List codes = new ArrayList(); - private Map map = new HashMap(); - private ValueSet focus; - + private List codes = new ArrayList(); + private IWorkerContext context; + private Set excludeKeys = new HashSet(); private ValueSetExpanderFactory factory; - - public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) { - super(); - this.context = context; - this.factory = factory; - } - - @Override - public ValueSetExpansionOutcome expand(ValueSet source) { + private ValueSet focus; - try { - focus = source.copy(); - focus.setExpansion(new ValueSet.ValueSetExpansionComponent()); - focus.getExpansion().setTimestampElement(DateTimeType.now()); - focus.getExpansion().setIdentifier(Factory.createUUID()); + private Map map = new HashMap(); - if (source.hasCompose()) - handleCompose(source.getCompose(), focus.getExpansion().getParameter()); - - for (ValueSetExpansionContainsComponent c : codes) { - if (map.containsKey(key(c))) { - focus.getExpansion().getContains().add(c); - } - } - return new ValueSetExpansionOutcome(focus, null); - } catch (RuntimeException e) { - // TODO: we should put something more specific instead of just Exception below, since - // it swallows bugs.. what would be expected to be caught there? - throw e; - } catch (Exception e) { - // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set - // that might fail too, but it might not, later. - return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage()); - } - } - - private void handleCompose(ValueSetComposeComponent compose, List params) throws TerminologyServiceException, ETooCostly, FileNotFoundException, IOException { - for (UriType imp : compose.getImport()) - importValueSet(imp.getValue(), params); - for (ConceptSetComponent inc : compose.getInclude()) - includeCodes(inc, params); - for (ConceptSetComponent inc : compose.getExclude()) - excludeCodes(inc, params); - - } - - private void importValueSet(String value, List params) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException { - if (value == null) - throw new TerminologyServiceException("unable to find value set with no identity"); - ValueSet vs = context.fetchResource(ValueSet.class, value); - if (vs == null) - throw new TerminologyServiceException("Unable to find imported value set "+value); - ValueSetExpansionOutcome vso = factory.getExpander().expand(vs); - if (vso.getService() != null) - throw new TerminologyServiceException("Unable to expand imported value set "+value); - if (vs.hasVersion()) - if (!existsInParams(params, "version", new UriType(vs.getUrl()+"?version="+vs.getVersion()))) - params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl()+"?version="+vs.getVersion()))); - for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) { - if (!existsInParams(params, p.getName(), p.getValue())) - params.add(p); - } - - for (ValueSetExpansionContainsComponent c : vso.getValueset().getExpansion().getContains()) { - addCode(c.getSystem(), c.getCode(), c.getDisplay()); - } - } - - private boolean existsInParams(List params, String name, Type value) { - for (ValueSetExpansionParameterComponent p : params) { - if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false)) - return true; - } - return false; - } - - private void includeCodes(ConceptSetComponent inc, List params) throws TerminologyServiceException, ETooCostly { - CodeSystem cs = context.fetchCodeSystem(inc.getSystem()); - if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) { - addCodes(context.expandVS(inc), params); - return; - } - - if (cs == null) - throw new TerminologyServiceException("unable to find code system "+inc.getSystem().toString()); - if (cs.getContent() != CodeSystemContentMode.COMPLETE) - throw new TerminologyServiceException("Code system "+inc.getSystem().toString()+" is incomplete"); - if (cs.hasVersion()) - if (!existsInParams(params, "version", new UriType(cs.getUrl()+"?version="+cs.getVersion()))) - params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl()+"?version="+cs.getVersion()))); - if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) { - // special case - add all the code system - for (ConceptDefinitionComponent def : cs.getConcept()) { - addCodeAndDescendents(cs, inc.getSystem(), def); - } - } - - for (ConceptReferenceComponent c : inc.getConcept()) { - addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay()); - } - if (inc.getFilter().size() > 1) - throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets - if (inc.getFilter().size() == 1) { - ConceptSetFilterComponent fc = inc.getFilter().get(0); - if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) { - // special: all non-abstract codes in the target code system under the value - ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue()); - if (def == null) - throw new TerminologyServiceException("Code '"+fc.getValue()+"' not found in system '"+inc.getSystem()+"'"); - addCodeAndDescendents(cs, inc.getSystem(), def); - } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) { - ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue()); - if (def != null) { - if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) { - if (def.getDisplay().contains(fc.getValue())) { - addCode(inc.getSystem(), def.getCode(), def.getDisplay()); - } - } - } - } else - throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet"); - } - } - - private void addCodes(ValueSetExpansionComponent expand, List params) throws ETooCostly { - if (expand.getContains().size() > 500) - throw new ETooCostly("Too many codes to display (>"+Integer.toString(expand.getContains().size())+")"); - for (ValueSetExpansionParameterComponent p : expand.getParameter()) { - if (!existsInParams(params, p.getName(), p.getValue())) - params.add(p); - } - - for (ValueSetExpansionContainsComponent c : expand.getContains()) { - addCode(c.getSystem(), c.getCode(), c.getDisplay()); - } - } - - private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def) { - if (!CodeSystemUtilities.isDeprecated(cs, def)) { - if (!CodeSystemUtilities.isAbstract(cs, def)) - addCode(system, def.getCode(), def.getDisplay()); - for (ConceptDefinitionComponent c : def.getConcept()) - addCodeAndDescendents(cs, system, c); - } - } - - private void excludeCodes(ConceptSetComponent inc, List params) throws TerminologyServiceException { - CodeSystem cs = context.fetchCodeSystem(inc.getSystem().toString()); - if (cs == null) - throw new TerminologyServiceException("unable to find value set "+inc.getSystem().toString()); - if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) { - // special case - add all the code system -// for (ConceptDefinitionComponent def : cs.getDefine().getConcept()) { -//!!!! addCodeAndDescendents(inc.getSystem(), def); -// } - } - - - for (ConceptReferenceComponent c : inc.getConcept()) { - // we don't need to check whether the codes are valid here- they can't have gotten into this list if they aren't valid - map.remove(key(inc.getSystem(), c.getCode())); - } - if (inc.getFilter().size() > 0) - throw new NotImplementedException("not done yet"); - } - - - private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException { - ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code); - if (def == null) - throw new TerminologyServiceException("Unable to find code '"+code+"' in code system "+cs.getUrl()); - return def.getDisplay(); - } - - private ConceptDefinitionComponent getConceptForCode(List clist, String code) { - for (ConceptDefinitionComponent c : clist) { - if (code.equals(c.getCode())) - return c; - ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code); - if (v != null) - return v; - } - return null; - } - - private String key(ValueSetExpansionContainsComponent c) { - return key(c.getSystem(), c.getCode()); - } - - private String key(String uri, String code) { - return "{"+uri+"}"+code; + public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) { + super(); + this.context = context; + this.factory = factory; } private void addCode(String system, String code, String display) { ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent(); n.setSystem(system); - n.setCode(code); - n.setDisplay(display); - String s = key(n); - if (!map.containsKey(s)) { - codes.add(n); - map.put(s, n); - } - } + n.setCode(code); + n.setDisplay(display); + String s = key(n); + if (!map.containsKey(s) && !excludeKeys.contains(s)) { + codes.add(n); + map.put(s, n); + } + } + + private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def) { + if (!CodeSystemUtilities.isDeprecated(cs, def)) { + if (!CodeSystemUtilities.isAbstract(cs, def)) + addCode(system, def.getCode(), def.getDisplay()); + for (ConceptDefinitionComponent c : def.getConcept()) + addCodeAndDescendents(cs, system, c); + } + } + + private void addCodes(ValueSetExpansionComponent expand, List params) throws ETooCostly { + if (expand.getContains().size() > 500) + throw new ETooCostly("Too many codes to display (>" + Integer.toString(expand.getContains().size()) + ")"); + for (ValueSetExpansionParameterComponent p : expand.getParameter()) { + if (!existsInParams(params, p.getName(), p.getValue())) + params.add(p); + } + + for (ValueSetExpansionContainsComponent c : expand.getContains()) { + addCode(c.getSystem(), c.getCode(), c.getDisplay()); + } + } + + private void excludeCode(String theSystem, String theCode) { + ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent(); + n.setSystem(theSystem); + n.setCode(theCode); + String s = key(n); + excludeKeys.add(s); + } + + private void excludeCodes(ConceptSetComponent inc, List params) throws TerminologyServiceException { + if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) { + return; + } + + CodeSystem cs = context.fetchCodeSystem(inc.getSystem()); + if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) { + excludeCodes(context.expandVS(inc), params); + return; + } + + for (ConceptReferenceComponent c : inc.getConcept()) { + excludeCode(inc.getSystem(), c.getCode()); + } + + if (inc.getFilter().size() > 0) + throw new NotImplementedException("not done yet"); + } + + private void excludeCodes(ValueSetExpansionComponent expand, List params) { + for (ValueSetExpansionContainsComponent c : expand.getContains()) { + excludeCode(c.getSystem(), c.getCode()); + } + } + + private boolean existsInParams(List params, String name, Type value) { + for (ValueSetExpansionParameterComponent p : params) { + if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false)) + return true; + } + return false; + } + + @Override + public ValueSetExpansionOutcome expand(ValueSet source) { + + try { + focus = source.copy(); + focus.setExpansion(new ValueSet.ValueSetExpansionComponent()); + focus.getExpansion().setTimestampElement(DateTimeType.now()); + focus.getExpansion().setIdentifier(Factory.createUUID()); + + if (source.hasCompose()) + handleCompose(source.getCompose(), focus.getExpansion().getParameter()); + + for (ValueSetExpansionContainsComponent c : codes) { + if (map.containsKey(key(c))) { + focus.getExpansion().getContains().add(c); + } + } + return new ValueSetExpansionOutcome(focus, null); + } catch (RuntimeException e) { + // TODO: we should put something more specific instead of just Exception below, since + // it swallows bugs.. what would be expected to be caught there? + throw e; + } catch (Exception e) { + // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set + // that might fail too, but it might not, later. + return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage()); + } + } + + private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException { + ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code); + if (def == null) + throw new TerminologyServiceException("Unable to find code '" + code + "' in code system " + cs.getUrl()); + return def.getDisplay(); + } + + private ConceptDefinitionComponent getConceptForCode(List clist, String code) { + for (ConceptDefinitionComponent c : clist) { + if (code.equals(c.getCode())) + return c; + ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code); + if (v != null) + return v; + } + return null; + } + + private void handleCompose(ValueSetComposeComponent compose, List params) throws TerminologyServiceException, ETooCostly, FileNotFoundException, IOException { + // Exclude comes first because we build up a map of things to exclude + for (ConceptSetComponent inc : compose.getExclude()) + excludeCodes(inc, params); + for (UriType imp : compose.getImport()) + importValueSet(imp.getValue(), params); + for (ConceptSetComponent inc : compose.getInclude()) + includeCodes(inc, params); + + } + + private void importValueSet(String value, List params) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException { + if (value == null) + throw new TerminologyServiceException("unable to find value set with no identity"); + ValueSet vs = context.fetchResource(ValueSet.class, value); + if (vs == null) + throw new TerminologyServiceException("Unable to find imported value set " + value); + ValueSetExpansionOutcome vso = factory.getExpander().expand(vs); + if (vso.getService() != null) + throw new TerminologyServiceException("Unable to expand imported value set " + value); + if (vs.hasVersion()) + if (!existsInParams(params, "version", new UriType(vs.getUrl() + "?version=" + vs.getVersion()))) + params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "?version=" + vs.getVersion()))); + for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) { + if (!existsInParams(params, p.getName(), p.getValue())) + params.add(p); + } + + for (ValueSetExpansionContainsComponent c : vso.getValueset().getExpansion().getContains()) { + addCode(c.getSystem(), c.getCode(), c.getDisplay()); + } + } + + private void includeCodes(ConceptSetComponent inc, List params) throws TerminologyServiceException, ETooCostly { + CodeSystem cs = context.fetchCodeSystem(inc.getSystem()); + if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) { + addCodes(context.expandVS(inc), params); + return; + } + + if (cs == null) + throw new TerminologyServiceException("unable to find code system " + inc.getSystem().toString()); + if (cs.getContent() != CodeSystemContentMode.COMPLETE) + throw new TerminologyServiceException("Code system " + inc.getSystem().toString() + " is incomplete"); + if (cs.hasVersion()) + if (!existsInParams(params, "version", new UriType(cs.getUrl() + "?version=" + cs.getVersion()))) + params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl() + "?version=" + cs.getVersion()))); + if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) { + // special case - add all the code system + for (ConceptDefinitionComponent def : cs.getConcept()) { + addCodeAndDescendents(cs, inc.getSystem(), def); + } + } + + for (ConceptReferenceComponent c : inc.getConcept()) { + addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay()); + } + if (inc.getFilter().size() > 1) + throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets + if (inc.getFilter().size() == 1) { + ConceptSetFilterComponent fc = inc.getFilter().get(0); + if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) { + // special: all non-abstract codes in the target code system under the value + ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue()); + if (def == null) + throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'"); + addCodeAndDescendents(cs, inc.getSystem(), def); + } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) { + ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue()); + if (def != null) { + if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) { + if (def.getDisplay().contains(fc.getValue())) { + addCode(inc.getSystem(), def.getCode(), def.getDisplay()); + } + } + } + } else + throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet"); + } + } + + private String key(String uri, String code) { + return "{" + uri + "}" + code; + } + + private String key(ValueSetExpansionContainsComponent c) { + return key(c.getSystem(), c.getCode()); + } - }